From 32b6e41b97e8bbda758a5522d9cb3080b00c38bc Mon Sep 17 00:00:00 2001 From: mrbesen Date: Fri, 19 Jul 2019 13:47:49 +0200 Subject: [PATCH] renamed to main.py; added json output, select all, color --- .gitignore | 6 +- Pipfile | 1 + src/TgStats.py | 183 ---------------------------------- src/main.py | 259 +++++++++++++++++++++++++++++++++++++++++++++++++ src/out.py | 52 ++++++++++ 5 files changed, 317 insertions(+), 184 deletions(-) delete mode 100755 src/TgStats.py create mode 100755 src/main.py create mode 100644 src/out.py diff --git a/.gitignore b/.gitignore index fe01b5e..907859e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ config.ini *.session Pipfile.lock -.idea/ \ No newline at end of file +.idea/ +*.csv +*.json +*.xml +*.sqlite \ No newline at end of file diff --git a/Pipfile b/Pipfile index 2f7f75a..bfe792c 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ verify_ssl = true httplib2 = "*" cryptg = "*" telethon = "*" +colorama = "*" [requires] python_version = "3.6" diff --git a/src/TgStats.py b/src/TgStats.py deleted file mode 100755 index 7592aba..0000000 --- a/src/TgStats.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -from telethon import TelegramClient, sync -import configparser, math, datetime - -dialogs = [] # list of known chats -me = None # the own entity -client = None -output = None # "csv", "stdout", "yaml", "json"... - - -def printDialog(interid, d): # debug only - print('{0:3d}| {1:14d} | {2:30} | {3:1}'.format(interid, d.id, d.name, d.pinned)) - - -def getUsernamebyID(userid): - for d in dialogs: - if d.id == userid: - return d.name - return "Unknown" # refresh dialog cache? - - -class Stat: - name = None - - def __init__(self, name): - self.name = name - - def addMsg(self, msgnum, msg, chat): - pass - - def getValue(self, count): - pass - - -class Countable(Stat): - acc = 0 - mult = 0 - min = 1 << 31 - max = 0 - - def parse(self, val): - return str(val) - - def getValue(self, count): - return "Min: " + self.parse(self.min) + " Max: " + self.parse(self.max) + " Gesamt: " \ - + self.parse(self.acc) + " Avg: " + self.parse(self.acc / count) + " Geom.Avg: " + self.parse(math.sqrt(self.mult)) - - def addMsg(self, msgnum, msg, chat): - count = self.count(msgnum, msg, chat) - if count > self.max: - self.max = count - if count < self.min: - self.min = count - self.acc = self.acc + count - self.mult = self.mult + (count * count) - - def count(self, msgnum, msg, chat): - pass - - -class CharCount(Countable): - - def __init__(self, name: str = "CharCount"): - super().__init__(name) - - def count(self, msgnum, msg, chat): - if msg.message != None: - return len(msg.message) - return 0 - - -class Dist(Countable): - prev = 0 - - def __init__(self, name: str = "Distanz(s)"): - super().__init__(name) - - def count(self, msgnum, msg, chat): - if msg.date != None: - old = self.prev - self.prev = msg.date - if old == 0: - return 0 - return int((old - msg.date).total_seconds()) - - def parse(self, val): - return str(datetime.timedelta(seconds=val)) - - -class DiscreteCount(Stat): - list = {} - - def getValue(self, count): - out = '' - # sorted_ = sorted(list.items(), key=lambda kv: kv[1]) - for name, count in self.list.items(): - if len(out) > 0: - out = out + ", " - out = out + name + ": " + str(count) # TODO: add percantage - return out; - - -class UserCount(DiscreteCount): - - def __init__(self, name: str = "Msg/User"): - super().__init__(name) - - def addMsg(self, msgnum, msg, chat): - global me - fromid = msg.from_id - self.list[fromid] = self.list.get(fromid, 0) + 1 - - def getValue(self, count): - # replace ids with names - newdict = {} - for id, count in self.list.items(): - newdict[getUsernamebyID(id)] = count - - self.list = newdict - return super().getValue(count) - -stats = { - CharCount(), - Dist(), - UserCount() -} - -# ================================== -# MAIN program - -# read config -try: - config = configparser.ConfigParser() - config.read('config.ini') - api_id = int(config.get('Main', 'api_id')) - api_hash = config.get('Main', 'api_hash') - session_name = config.get('Main', 'user') -except (configparser.NoSectionError, configparser.NoOptionError, ValueError): - print('invalid config.ini') - exit(3) - -client = TelegramClient(session_name, api_id, api_hash) -client.start() - -me = client.get_me() -print('me.id: ', me.id) - -# get dialogs -dialogs = client.get_dialogs() -dialogCount = len(dialogs) -print(dialogCount, ' Chats loaded.') - -print( - ' Internal ID | Username | pinned\n———+————————————————+————————————————————————————————+———————') -interid = 0 -for d in dialogs: - printDialog(interid, d) - interid = interid + 1 - -get = int(input("Please Enter Chat ID: ")) -if get < 0 or get >= dialogCount: - print("Unknown Chat ID!") - exit(1) - -selectedDialog = dialogs[get] - -print("selected: \"", selectedDialog.name, '"; retriving chat!', sep='') - -# chat = client.get_messages(selectedDialog) -chat = client.iter_messages(selectedDialog, limit=None) - -# run messure -msgnum = 0 -for msg in chat: - if msg != None: - msgnum = msgnum + 1 - for stat in stats: - stat.addMsg(msgnum, msg, selectedDialog) - -print(msgnum, 'Nachrichten Analysiert') - -for stat in stats: - print(stat.name, ': ', stat.getValue(msgnum), sep='') diff --git a/src/main.py b/src/main.py new file mode 100755 index 0000000..91efacd --- /dev/null +++ b/src/main.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +from telethon import TelegramClient, sync +from timeit import default_timer as timer +import configparser +import out +import math, datetime +import colorama +from colorama import Fore, Style + +class Stat: + name = None + + def __init__(self, name): + self.name = name + + def addMsg(self, msgnum, msg, chat): + pass + + def getValue(self, count): + out = '' + all = self.getAll(count) + if len(all) == 0: + return '' + if len(all) == 1: + return next(iter(all)) + for name, val in all.items(): + if out: + out = out + ", " + out = out + name + ": " + str(val) + return out + + def getAll(self, count): + pass + + def parse(self, val): + return str(val) + + +class Countable(Stat): + acc = 0 + mult = 0 + min = 1 << 31 + max = 0 + + """def getValue(self, count): + return "Min: " + self.getMin() + " Max: " + self.getMax() + " Gesamt: " \ + + self.getAcc() + " Avg: " + self.getAvg(count) + " Geom.Avg: " + self.getGAvg() + """ + def getMin(self): + return self.parse(self.min) + + def getMax(self): + return self.parse(self.max) + + def getAcc(self): + return self.parse(self.acc) + + def getAvg(self, count): + return self.parse(int(self.acc / count)) + + def getGAvg(self): + return self.parse(math.sqrt(self.mult)) + + def getAll(self, count): + return {'min': self.getMin(), 'max': self.getMax(), 'count': self.getAcc(), 'avg': self.getAvg(count), 'gavg': self.getGAvg()} + + def addMsg(self, msgnum, msg, chat): + count = self.count(msgnum, msg, chat) + if count > self.max: + self.max = count + if count < self.min: + self.min = count + self.acc = self.acc + count + self.mult = self.mult + (count * count) + + def count(self, msgnum, msg, chat): + pass + + +class CharCount(Countable): + + def __init__(self, name: str = "CharCount"): + super().__init__(name) + + def count(self, msgnum, msg, chat): + if msg.message != None: + return len(msg.message) + return 0 + + +class Dist(Countable): + prev = 0 + + def __init__(self, name: str = "Distanz"): + super().__init__(name) + + def count(self, msgnum, msg, chat): + if msg.date != None: + old = self.prev + self.prev = msg.date + if old == 0: + return 0 + return int((old - msg.date).total_seconds()) + + def parse(self, val): + return str(datetime.timedelta(seconds=val)) + + def getAvg(self, count): + if count < 2: + return -1 + return self.parse(int(self.acc / (count-1))) # da hier die abstände zwischen nachrichten betrachtet werden ist es um 1 kleiner + + +class DiscreteCount(Stat): + list = {} + + """def getValue(self, count): + out = '' + # sorted_ = sorted(list.items(), key=lambda kv: kv[1]) + for name, count in self.list.items(): + if len(out) > 0: + out = out + ", " + out = out + name + ": " + str(count) + return out;""" + + def getAll(self, count): + return self.list + + +class UserCount(DiscreteCount): + + def __init__(self, name: str = "Msg/User"): + super().__init__(name) + + def addMsg(self, msgnum, msg, chat): + global me + fromid = msg.from_id + self.list[fromid] = self.list.get(fromid, 0) + 1 + + + def getAll(self, count): + # replace ids with names + newdict = {} + for id, count in self.list.items(): + newdict[getUsernamebyID(id)] = count + + return newdict + + +def printDialog(interid, d): # debug only + color = '' + if interid & 1: + color = Fore.LIGHTBLACK_EX + print(color + '{0:3d}| {1:14d} | {2:30} | {3:1}'.format(interid, d.id, d.name, d.pinned) + Fore.RESET) + + +def getUsernamebyID(userid): + global dialogs + for d in dialogs: + if d.id == userid: + return d.name + return "Unknown" # refresh dialog cache? + +def analyseChat(dialog, output): + print(Fore.GREEN + 'Lade kompletten Chat: ' + Fore.RESET, dialog.name, sep='') + + # chat = client.get_messages(selectedDialog) + chat = client.iter_messages(dialog, limit=None) + + # run messure + msgnum = 0 + for msg in chat: + if msg != None: + msgnum = msgnum + 1 + for stat in stats: + stat.addMsg(msgnum, msg, dialog) + + print(Fore.BLUE, msgnum, Fore.MAGENTA + 'Nachrichten Analysiert' + Fore.RESET) + + # write to output + output.print(stats, msgnum, dialog) + +# ================================== +# MAIN program +if __name__ == "__main__": + stats = { + CharCount(), + Dist(), + UserCount() + } + + outputs = { + "stdout": out.STDOUT(), + "json": out.jsonOut() + # TODO: add csv,xml,yaml maybe sqlite or mysql? + } + + # read config + try: + config = configparser.ConfigParser() + config.read('config.ini') + api_id = int(config.get('Main', 'api_id')) + api_hash = config.get('Main', 'api_hash') + session_name = config.get('Main', 'user') + except (configparser.NoSectionError, configparser.NoOptionError, ValueError): + print('invalid config.ini') + exit(3) + + client = TelegramClient(session_name, api_id, api_hash) + client.start() + + me = client.get_me() + print('me.id: ', me.id) + + # get dialogs + dialogs = client.get_dialogs() + dialogCount = len(dialogs) + print(Fore.YELLOW, dialogCount, Fore.GREEN + 'Chats geladen.' + Fore.RESET) + + #select output + while True: + uinput = input("Bitte Ausgabemethode wählen: [stdout,json] (stdout): ").strip().lower(); + if uinput == "": + uinput = "stdout" + try: + out = outputs[uinput] + break + except KeyError: + pass # try again + + outfile = None + if out.needsFilename: + outfile = input("Bitte Dateinamen eingeben: ").strip() + out.open(outfile) + + #select chat + print(' ID | Internal ID | Username | pinned\n———+————————————————+————————————————————————————————+———————') + interid = 0 + for d in dialogs: + printDialog(interid, d) + interid = interid + 1 + + uinput = input('Bitte ' + Fore.BLUE + 'chatid' + Fore.RESET + ' oder "' + Fore.BLUE + 'all' + Fore.RESET + '" eingeben: ').strip().lower() # TODO: select more than one or all + + start = timer() + if uinput == 'all': + for d in dialogs: + analyseChat(d, out) + else: + get = int(uinput) + if get < 0 or get >= dialogCount: #TODO: detect user by username, name, tg-id + print("unbekannte Chat ID!") + exit(1) + + analyseChat(dialogs[get], out) + out.close() + took = datetime.timedelta(seconds=(timer()-start)) + took = datetime.timedelta(seconds=int(took.total_seconds())) + print(Fore.GREEN + "Fertig. Benötigte Zeit: " + Fore.YELLOW, took, Style.RESET_ALL) diff --git a/src/out.py b/src/out.py new file mode 100644 index 0000000..d5307aa --- /dev/null +++ b/src/out.py @@ -0,0 +1,52 @@ +import json + +class Output: + name = None + needsFilename = False + + def __init__(self, name): + super().__init__() + self.name = name + + def open(self, file: str = None): + pass + + def close(self): + pass + + def print(self, stats, count, dialog): + pass + + +class STDOUT(Output): + + def __init__(self, name: str = "stdout"): + super().__init__(name) + + def print(self, stats, count, dialog): + for stat in stats: + print(stat.name, ': ', stat.getValue(count), sep='') + +class jsonOut(Output): + + file = None + needsFilename = True + outbuff = {} + + def __init__(self, name: str = "json"): + super().__init__(name) + + def open(self, file): + if file is None: + file = "out.json" + self.file = open(file, "w") + + def close(self): + json.dump(self.outbuff, self.file, indent=4) + self.file.close() + + def print(self, stats, count, dialog): + jsonpre = {"count": count} + for stat in stats: + jsonpre[stat.name] = stat.getAll(count) + self.outbuff[dialog.id] = jsonpre