raw
logotron_genesis.kv     1 #!/usr/bin/python
logotron_genesis.kv     2 
logotron_genesis.kv     3 ##############################################################################
logotron_genesis.kv     4 import psycopg2, psycopg2.extras
logotron_genesis.kv     5 import psycopg2.extensions
logotron_genesis.kv     6 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
logotron_genesis.kv     7 psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
logotron_genesis.kv     8 import re
logotron_genesis.kv     9 import time
logotron_genesis.kv    10 import datetime
logotron_genesis.kv    11 from datetime import datetime
logotron_genesis.kv    12 import sys
logotron_genesis.kv    13 import os
logotron_genesis.kv    14 
logotron_genesis.kv    15 # Debug Knob
logotron_genesis.kv    16 DB_DEBUG = False
logotron_genesis.kv    17 ##############################################################################
logotron_genesis.kv    18 
logotron_genesis.kv    19 ##############################################################################
logotron_genesis.kv    20 db = psycopg2.connect("dbname=nsalog user=nsabot") ## CHANGE THESE
logotron_genesis.kv    21 
logotron_genesis.kv    22 def close_db():
logotron_genesis.kv    23     db.close()
logotron_genesis.kv    24 
logotron_genesis.kv    25 def exec_db(query, args=()):
logotron_genesis.kv    26     cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv    27     if (DB_DEBUG): print "query: '{0}'".format(query)
logotron_genesis.kv    28     if (DB_DEBUG): print "args: '{0}'".format(args)
logotron_genesis.kv    29     if (DB_DEBUG): print "EXEC:"
logotron_genesis.kv    30     cur.execute(query, args)
logotron_genesis.kv    31 
logotron_genesis.kv    32 def rollback_db():
logotron_genesis.kv    33     cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv    34     cur.execute("ROLLBACK")
logotron_genesis.kv    35     db.commit()
logotron_genesis.kv    36 
logotron_genesis.kv    37 def commit_db():
logotron_genesis.kv    38     cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv    39     db.commit()
logotron_genesis.kv    40 
logotron_genesis.kv    41 ##############################################################################
logotron_genesis.kv    42 
logotron_genesis.kv    43 # Eat individual line of a Phf-style log dump
logotron_genesis.kv    44 def eat_logline(line, chan, era):
logotron_genesis.kv    45     match = re.search("(\d+)\;(\d+)\;([^;]+)\;(.*$)", line)
logotron_genesis.kv    46     if match:
logotron_genesis.kv    47         g = match.groups()
logotron_genesis.kv    48         self_speak = False
logotron_genesis.kv    49 
logotron_genesis.kv    50         try:
logotron_genesis.kv    51             idx        = int(g[0]) # Serial Number of Log Line
logotron_genesis.kv    52             time       = int(g[1]) # Unix Epochal Time of Log Line
logotron_genesis.kv    53         except Exception, e:
logotron_genesis.kv    54             print("Malformed Line! '" + line +"' ! : " + e)
logotron_genesis.kv    55             close_db()
logotron_genesis.kv    56             exit(1)
logotron_genesis.kv    57 
logotron_genesis.kv    58         speaker    = g[2] # Name of Speaker
logotron_genesis.kv    59         payload    = g[3] # Payload (remainder of line)
logotron_genesis.kv    60 
logotron_genesis.kv    61         ## If spoken line is of form "* user ..." :
logotron_genesis.kv    62         if speaker == "*":
logotron_genesis.kv    63             spl = payload.split(' ', 1)
logotron_genesis.kv    64             speaker = spl[0]
znc2tmsr_etc.kv        65             try:
znc2tmsr_etc.kv        66                 payload = spl[1]
znc2tmsr_etc.kv        67             except IndexError:
znc2tmsr_etc.kv        68                 payload = ""
logotron_genesis.kv    69             self_speak = True
znc2tmsr_etc.kv        70         
znc2tmsr_etc.kv        71         ## Handle uniturds using the phf algorithm 
searchpg_oldlinks.kv   72         try:
searchpg_oldlinks.kv   73             payload = payload.decode("utf-8")
searchpg_oldlinks.kv   74         except UnicodeDecodeError:
searchpg_oldlinks.kv   75             payload = payload.decode('latin-1')
znc2tmsr_etc.kv        76         
logotron_genesis.kv    77         ## Put in DB:
logotron_genesis.kv    78         try:
logotron_genesis.kv    79             exec_db('''insert into loglines (idx, t, chan, era, speaker, self, payload)
logotron_genesis.kv    80             values (%s, %s, %s, %s, %s, %s, %s) ; ''',
logotron_genesis.kv    81                     [int(idx), datetime.fromtimestamp(time), str(chan), int(era), str(speaker),
searchpg_oldlinks.kv   82                      bool(self_speak), payload])
logotron_genesis.kv    83             commit_db()
logotron_genesis.kv    84         except psycopg2.IntegrityError as e:
logotron_genesis.kv    85             rollback_db()
logotron_genesis.kv    86             print "Dupe Ignored, Idx=", idx
logotron_genesis.kv    87     else:
logotron_genesis.kv    88         print("Malformed Line! '" + line +"' !")
logotron_genesis.kv    89         close_db()
logotron_genesis.kv    90         exit(1)
logotron_genesis.kv    91 
logotron_genesis.kv    92 
logotron_genesis.kv    93 # Eat Phf-style log dump at given path
logotron_genesis.kv    94 def eat_dump(path, chan, era):
logotron_genesis.kv    95     with open(path) as fp:
logotron_genesis.kv    96         for line in fp:
logotron_genesis.kv    97             eat_logline(line, chan, era)
logotron_genesis.kv    98 
logotron_genesis.kv    99 
logotron_genesis.kv   100 ##############################################################################
logotron_genesis.kv   101 
logotron_genesis.kv   102 if (len(sys.argv) == 4):
logotron_genesis.kv   103     logdump = sys.argv[1] # Path to Phf-style log dump
logotron_genesis.kv   104     chan    = sys.argv[2] # Chan Name
logotron_genesis.kv   105     era     = sys.argv[3] # Era (integer)
logotron_genesis.kv   106     # Eat:
logotron_genesis.kv   107     eat_dump(logdump, chan, era)
logotron_genesis.kv   108     close_db()
logotron_genesis.kv   109 else:
logotron_genesis.kv   110     print "Usage: ./eat_dump LOGFILE CHAN ERA"
logotron_genesis.kv   111     exit(0)
logotron_genesis.kv   112 
logotron_genesis.kv   113 ##############################################################################