logotron_genesis.kv 1
logotron_genesis.kv 2
logotron_genesis.kv 3
logotron_genesis.kv 4 import ConfigParser, sys
logotron_genesis.kv 5 import psycopg2, psycopg2.extras
logotron_genesis.kv 6 import psycopg2.extensions
logotron_genesis.kv 7 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
logotron_genesis.kv 8 psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
logotron_genesis.kv 9 import time
logotron_genesis.kv 10 import datetime
logotron_genesis.kv 11 from datetime import timedelta
logotron_genesis.kv 12 import sys
logotron_genesis.kv 13 reload(sys)
logotron_genesis.kv 14 sys.setdefaultencoding('utf8')
logotron_genesis.kv 15 import os
logotron_genesis.kv 16 import threading
logotron_genesis.kv 17 import re
logotron_genesis.kv 18 from datetime import datetime
logotron_genesis.kv 19 from urlparse import urljoin
logotron_genesis.kv 20 from flask import Flask, request, session, url_for, redirect, \
logotron_genesis.kv 21 render_template, abort, g, flash, _app_ctx_stack, make_response, \
logotron_genesis.kv 22 jsonify
logotron_genesis.kv 23 from flask import Flask
logotron_genesis.kv 24 from flask.ext.cache import Cache
logotron_genesis.kv 25
logotron_genesis.kv 26
logotron_genesis.kv 27
logotron_genesis.kv 28
logotron_genesis.kv 29 if len(sys.argv[1:]) != 1:
logotron_genesis.kv 30
logotron_genesis.kv 31 print sys.argv[0] + " CONFIG"
logotron_genesis.kv 32 exit(0)
logotron_genesis.kv 33
logotron_genesis.kv 34
logotron_genesis.kv 35 config_path = os.path.abspath(sys.argv[1])
logotron_genesis.kv 36 cfg = ConfigParser.ConfigParser()
logotron_genesis.kv 37 cfg.readfp(open(config_path))
logotron_genesis.kv 38
logotron_genesis.kv 39 try:
logotron_genesis.kv 40
logotron_genesis.kv 41 Nick = cfg.get("irc", "nick")
logotron_genesis.kv 42 Channels = [x.strip() for x in cfg.get("irc", "chans").split(',')]
logotron_genesis.kv 43 Bots = [x.strip() for x in cfg.get("logotron", "bots").split(',')]
logotron_genesis.kv 44 Bots.append(Nick)
logotron_genesis.kv 45
logotron_genesis.kv 46 DB_Name = cfg.get("db", "db_name")
logotron_genesis.kv 47 DB_User = cfg.get("db", "db_user")
logotron_genesis.kv 48 DB_DEBUG = cfg.get("db", "db_debug")
logotron_genesis.kv 49
logotron_genesis.kv 50 Base_URL = cfg.get("logotron", "base_url")
logotron_genesis.kv 51 Era = int(cfg.get("logotron", "era"))
logotron_genesis.kv 52
logotron_genesis.kv 53 WWW_Port = int(cfg.get("logotron", "www_port"))
logotron_genesis.kv 54
logotron_genesis.kv 55 except Exception as e:
logotron_genesis.kv 56 print "Invalid config: ", e
logotron_genesis.kv 57 exit(1)
logotron_genesis.kv 58
logotron_genesis.kv 59
logotron_genesis.kv 60
logotron_genesis.kv 61
logotron_genesis.kv 62
logotron_genesis.kv 63 Default_Chan = Channels[0]
logotron_genesis.kv 64 Min_Query_Length = 3
logotron_genesis.kv 65 Max_Search_Results = 1000
logotron_genesis.kv 66
logotron_genesis.kv 67
logotron_genesis.kv 68 Date_Short_Format = "%Y-%m-%d"
logotron_genesis.kv 69
logotron_genesis.kv 70
logotron_genesis.kv 71 DEBUG = False
logotron_genesis.kv 72
logotron_genesis.kv 73
logotron_genesis.kv 74 app = Flask(__name__)
logotron_genesis.kv 75 cache = Cache(app,config={'CACHE_TYPE': 'simple'})
logotron_genesis.kv 76 app.config.from_object(__name__)
logotron_genesis.kv 77
logotron_genesis.kv 78 def get_db():
logotron_genesis.kv 79 db = getattr(g, 'db', None)
logotron_genesis.kv 80 if db is None:
logotron_genesis.kv 81 db = g.db = psycopg2.connect("dbname=%s user=%s" % (DB_Name, DB_User))
logotron_genesis.kv 82 return db
logotron_genesis.kv 83
logotron_genesis.kv 84 def close_db():
logotron_genesis.kv 85 if hasattr(g, 'db'):
logotron_genesis.kv 86 g.db.close()
logotron_genesis.kv 87
logotron_genesis.kv 88 @app.before_request
logotron_genesis.kv 89 def before_request():
logotron_genesis.kv 90 g.db = get_db()
logotron_genesis.kv 91
logotron_genesis.kv 92 @app.teardown_request
logotron_genesis.kv 93 def teardown_request(exception):
logotron_genesis.kv 94 close_db()
logotron_genesis.kv 95
logotron_genesis.kv 96 def query_db(query, args=(), one=False):
logotron_genesis.kv 97 cur = get_db().cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv 98 if (DB_DEBUG): print "query: '{0}'".format(query)
logotron_genesis.kv 99 cur.execute(query, args)
logotron_genesis.kv 100 rv = cur.fetchone() if one else cur.fetchall()
logotron_genesis.kv 101 if (DB_DEBUG): print "query res: '{0}'".format(rv)
logotron_genesis.kv 102 return rv
logotron_genesis.kv 103
logotron_genesis.kv 104 def exec_db(query, args=()):
logotron_genesis.kv 105 cur = get_db().cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv 106 if (DB_DEBUG): print "query: '{0}'".format(query)
logotron_genesis.kv 107 if (DB_DEBUG): print "args: '{0}'".format(args)
logotron_genesis.kv 108 if (DB_DEBUG): print "EXEC:"
logotron_genesis.kv 109 cur.execute(query, args)
logotron_genesis.kv 110
logotron_genesis.kv 111 def getlast_db():
logotron_genesis.kv 112 cur = get_db().cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv 113 cur.execute('select lastval()')
logotron_genesis.kv 114 return cur.fetchone()['lastval']
logotron_genesis.kv 115
logotron_genesis.kv 116 def commit_db():
logotron_genesis.kv 117 cur = get_db().cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv 118 g.db.commit()
logotron_genesis.kv 119
logotron_genesis.kv 120
logotron_genesis.kv 121
logotron_genesis.kv 122
logotron_genesis.kv 123 @app.errorhandler(404)
logotron_genesis.kv 124 def page_not_found(error):
logotron_genesis.kv 125 return redirect(url_for('log'))
logotron_genesis.kv 126
logotron_genesis.kv 127
logotron_genesis.kv 128
logotron_genesis.kv 129 html_escape_table = {
logotron_genesis.kv 130 "&": "&",
logotron_genesis.kv 131 '"': """,
logotron_genesis.kv 132 "'": "'",
logotron_genesis.kv 133 ">": ">",
logotron_genesis.kv 134 "<": "<",
logotron_genesis.kv 135 }
logotron_genesis.kv 136
logotron_genesis.kv 137 def html_escape(text):
logotron_genesis.kv 138 return "".join(html_escape_table.get(c,c) for c in text)
logotron_genesis.kv 139
logotron_genesis.kv 140
logotron_genesis.kv 141
logotron_genesis.kv 142
logotron_genesis.kv 143 def get_base():
logotron_genesis.kv 144 if DEBUG:
logotron_genesis.kv 145 return request.host_url
logotron_genesis.kv 146 return Base_URL
logotron_genesis.kv 147
logotron_genesis.kv 148
logotron_genesis.kv 149
logotron_genesis.kv 150 def line_url(l):
logotron_genesis.kv 151 return "{0}log/{1}/{2}#{3}".format(get_base(),
logotron_genesis.kv 152 l['chan'],
logotron_genesis.kv 153 l['t'].strftime(Date_Short_Format),
logotron_genesis.kv 154 l['idx'])
logotron_genesis.kv 155
logotron_genesis.kv 156 def gen_chanlist(selected_chan):
logotron_genesis.kv 157
logotron_genesis.kv 158 now = datetime.now()
logotron_genesis.kv 159
logotron_genesis.kv 160 s = """<table align="center" class="chantable"><tr>"""
logotron_genesis.kv 161 for chan in Channels:
logotron_genesis.kv 162 chan_formed = chan
logotron_genesis.kv 163 if chan == selected_chan:
logotron_genesis.kv 164 chan_formed = "<span class='highlight'>" + chan + "</span>"
logotron_genesis.kv 165 s += """<th><a href="{0}log/{1}">{2}</a></th>""".format(
logotron_genesis.kv 166 get_base(), chan, chan_formed)
logotron_genesis.kv 167 s += "</tr><tr>"
logotron_genesis.kv 168
logotron_genesis.kv 169 for chan in Channels:
logotron_genesis.kv 170
logotron_genesis.kv 171 last_time = query_db(
logotron_genesis.kv 172 '''select t, idx from loglines where chan=%s
logotron_genesis.kv 173 and idx = (select max(idx) from loglines where chan=%s) ;''',
logotron_genesis.kv 174 [chan, chan], one=True)
logotron_genesis.kv 175
logotron_genesis.kv 176 last_time_txt = ""
logotron_genesis.kv 177 if last_time != None:
logotron_genesis.kv 178 span = (now - last_time['t'])
logotron_genesis.kv 179 days = span.days
logotron_genesis.kv 180 hours = span.seconds/3600
logotron_genesis.kv 181 minutes = (span.seconds%3600)/60
logotron_genesis.kv 182
logotron_genesis.kv 183 if days != 0:
logotron_genesis.kv 184 last_time_txt += '%dd ' % days
logotron_genesis.kv 185 if hours != 0:
logotron_genesis.kv 186 last_time_txt += '%dh ' % hours
logotron_genesis.kv 187 if minutes != 0:
logotron_genesis.kv 188 last_time_txt += '%dm' % minutes
logotron_genesis.kv 189
logotron_genesis.kv 190 s += """<td><i><a href="{0}log/{1}/{2}
logotron_genesis.kv 191 get_base(),
logotron_genesis.kv 192 chan,
logotron_genesis.kv 193 last_time['t'].strftime(Date_Short_Format),
logotron_genesis.kv 194 last_time['idx'],
logotron_genesis.kv 195 last_time_txt)
logotron_genesis.kv 196
logotron_genesis.kv 197 else:
logotron_genesis.kv 198 last_time_txt = ""
logotron_genesis.kv 199 s += "<td></td>"
logotron_genesis.kv 200
logotron_genesis.kv 201 s += "</tr></table>"
logotron_genesis.kv 202 return s
logotron_genesis.kv 203
logotron_genesis.kv 204
logotron_genesis.kv 205
logotron_genesis.kv 206 app.jinja_env.globals.update(gen_chanlist=gen_chanlist)
logotron_genesis.kv 207
logotron_genesis.kv 208
logotron_genesis.kv 209
logotron_genesis.kv 210 tag_regex = re.compile("(<[^>]+>)")
logotron_genesis.kv 211
logotron_genesis.kv 212
logotron_genesis.kv 213
logotron_genesis.kv 214 def get_link_intervals(str):
logotron_genesis.kv 215 links = []
logotron_genesis.kv 216 span = []
logotron_genesis.kv 217 for match in tag_regex.finditer(str):
logotron_genesis.kv 218 span = match.span()
logotron_genesis.kv 219 links += [span]
logotron_genesis.kv 220 return links
logotron_genesis.kv 221
logotron_genesis.kv 222
logotron_genesis.kv 223
logotron_genesis.kv 224 def highlight_matches(strings, text):
logotron_genesis.kv 225 e = '(' + ('|'.join(strings)) + ')'
logotron_genesis.kv 226 return re.sub(e,
logotron_genesis.kv 227 r"""<span class='highlight'>\1</span>""",
logotron_genesis.kv 228 text,
logotron_genesis.kv 229 flags=re.I)
logotron_genesis.kv 230
logotron_genesis.kv 231
logotron_genesis.kv 232
logotron_genesis.kv 233
logotron_genesis.kv 234 def highlight_text(strings, text):
logotron_genesis.kv 235 result = ""
logotron_genesis.kv 236 last = 0
logotron_genesis.kv 237 for i in get_link_intervals(text):
logotron_genesis.kv 238 i_start, i_end = i
logotron_genesis.kv 239 result += highlight_matches(strings, text[last:i_start])
logotron_genesis.kv 240 result += text[i_start:i_end]
logotron_genesis.kv 241 last = i_end
logotron_genesis.kv 242 result += highlight_matches(strings, text[last:])
logotron_genesis.kv 243 return result
logotron_genesis.kv 244
logotron_genesis.kv 245
logotron_genesis.kv 246
logotron_genesis.kv 247 boxlinks_re = re.compile('\[\s*<a href="(http[^ \[\]]+)">[^ <]+</a>\s*\]\[([^\[\]]+)\]')
logotron_genesis.kv 248 stdlinks_re = re.compile('(http[^ \[\]]+)')
logotron_genesis.kv 249
logotron_genesis.kv 250
logotron_genesis.kv 251
multsel_and_datef... 252 def format_logline(l, highlights = [], select=[]):
logotron_genesis.kv 253 payload = html_escape(l['payload'])
logotron_genesis.kv 254
logotron_genesis.kv 255
logotron_genesis.kv 256 payload = re.sub(stdlinks_re, r'<a href="\1">\1</a>', payload)
logotron_genesis.kv 257
logotron_genesis.kv 258
logotron_genesis.kv 259 payload = re.sub(boxlinks_re, r'<a href="\1">\2</a>', payload)
logotron_genesis.kv 260
logotron_genesis.kv 261
logotron_genesis.kv 262 if highlights != []:
logotron_genesis.kv 263 payload = highlight_text(highlights, payload)
logotron_genesis.kv 264
logotron_genesis.kv 265 bot = ""
logotron_genesis.kv 266 if l['speaker'] in Bots:
logotron_genesis.kv 267 bot = " bot"
logotron_genesis.kv 268
multsel_and_datef... 269
multsel_and_datef... 270 dclass = l['speaker']
multsel_and_datef... 271
multsel_and_datef... 272
multsel_and_datef... 273 if select != []:
multsel_and_datef... 274 ss, se = select
multsel_and_datef... 275 if ss <= l['idx'] <= se:
multsel_and_datef... 276 dclass = "highlight"
multsel_and_datef... 277
logotron_genesis.kv 278
multsel_and_datef... 279 s = ("<div id='{0}' class='{6}{5}'>"
logotron_genesis.kv 280 "<a class='nick' title='{2}'"
logotron_genesis.kv 281 " href=\"{3}\">{1}</a>: {4}</div>").format(l['idx'],
logotron_genesis.kv 282 l['speaker'],
logotron_genesis.kv 283 l['t'],
logotron_genesis.kv 284 line_url(l),
logotron_genesis.kv 285 payload,
multsel_and_datef... 286 bot,
multsel_and_datef... 287 dclass)
logotron_genesis.kv 288 return s
logotron_genesis.kv 289
logotron_genesis.kv 290
logotron_genesis.kv 291 app.jinja_env.globals.update(format_logline=format_logline)
logotron_genesis.kv 292
logotron_genesis.kv 293
logotron_genesis.kv 294
logotron_genesis.kv 295 def generate_navbar(date, tail, chan):
logotron_genesis.kv 296 cur_day = datetime.strptime(date, Date_Short_Format)
logotron_genesis.kv 297 prev_day = cur_day - timedelta(days=1)
logotron_genesis.kv 298 prev_day_txt = prev_day.strftime(Date_Short_Format)
logotron_genesis.kv 299
logotron_genesis.kv 300 s = "<a href='{0}log/{1}/{2}'>← {2}</a>".format(
logotron_genesis.kv 301 get_base(),
logotron_genesis.kv 302 chan,
logotron_genesis.kv 303 prev_day_txt)
logotron_genesis.kv 304
logotron_genesis.kv 305 if not tail:
logotron_genesis.kv 306 next_day = cur_day + timedelta(days=1)
logotron_genesis.kv 307 next_day_txt = next_day.strftime(Date_Short_Format)
logotron_genesis.kv 308 s = s + " | <a href='{0}log/{1}/{2}'>{2} →</a>".format(
logotron_genesis.kv 309 get_base(),
logotron_genesis.kv 310 chan,
logotron_genesis.kv 311 next_day_txt)
logotron_genesis.kv 312
logotron_genesis.kv 313 return s
logotron_genesis.kv 314
logotron_genesis.kv 315
logotron_genesis.kv 316 app.jinja_env.globals.update(generate_navbar=generate_navbar)
logotron_genesis.kv 317
logotron_genesis.kv 318
logotron_genesis.kv 319 @app.route('/log/<chan>/<date>')
logotron_genesis.kv 320 @app.route('/log/<chan>', defaults={'date': None})
logotron_genesis.kv 321 @app.route('/log/', defaults={'chan': Default_Chan, 'date': None})
logotron_genesis.kv 322 @app.route('/log', defaults={'chan': Default_Chan, 'date': None})
logotron_genesis.kv 323 def log(chan, date):
logotron_genesis.kv 324
logotron_genesis.kv 325 if chan not in Channels:
logotron_genesis.kv 326 return redirect(url_for('log'))
logotron_genesis.kv 327
multsel_and_datef... 328
multsel_and_datef... 329 sel_start = request.args.get('ss', default = 0, type = int)
multsel_and_datef... 330 sel_end = request.args.get('se', default = 0, type = int)
multsel_and_datef... 331
logotron_genesis.kv 332
logotron_genesis.kv 333 now = datetime.now()
logotron_genesis.kv 334
logotron_genesis.kv 335
logotron_genesis.kv 336 tail = False
logotron_genesis.kv 337
logotron_genesis.kv 338
logotron_genesis.kv 339 if date == None:
logotron_genesis.kv 340 date = now.strftime(Date_Short_Format)
logotron_genesis.kv 341 tail = True
logotron_genesis.kv 342
logotron_genesis.kv 343
logotron_genesis.kv 344 try:
logotron_genesis.kv 345 day_start = datetime.strptime(date, Date_Short_Format)
logotron_genesis.kv 346 except Exception, e:
logotron_genesis.kv 347 return redirect(url_for('log'))
logotron_genesis.kv 348
logotron_genesis.kv 349
logotron_genesis.kv 350 day_end = day_start + timedelta(days=1)
logotron_genesis.kv 351
multsel_and_datef... 352
multsel_and_datef... 353 if day_end.day > now.day:
multsel_and_datef... 354 tail = True
multsel_and_datef... 355
logotron_genesis.kv 356
logotron_genesis.kv 357 lines = query_db(
logotron_genesis.kv 358 '''select * from loglines where chan=%s
logotron_genesis.kv 359 and t between %s and %s order by idx asc;''',
logotron_genesis.kv 360 [chan, day_start, day_end], one=False)
logotron_genesis.kv 361
logotron_genesis.kv 362
logotron_genesis.kv 363 return render_template('log.html',
logotron_genesis.kv 364 chan = chan,
logotron_genesis.kv 365 loglines = lines,
multsel_and_datef... 366 sel = (sel_start, sel_end),
logotron_genesis.kv 367 date = date,
logotron_genesis.kv 368 tail = tail)
logotron_genesis.kv 369
logotron_genesis.kv 370
logotron_genesis.kv 371
logotron_genesis.kv 372 Name_Chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"
logotron_genesis.kv 373
logotron_genesis.kv 374 def sanitize_speaker(s):
logotron_genesis.kv 375 return "".join([ch for ch in s if ch in Name_Chars])
logotron_genesis.kv 376
logotron_genesis.kv 377
logotron_genesis.kv 378 def re_escape(s):
logotron_genesis.kv 379 return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s)
logotron_genesis.kv 380
logotron_genesis.kv 381
logotron_genesis.kv 382 @app.route('/log-search')
logotron_genesis.kv 383 def logsearch():
logotron_genesis.kv 384
logotron_genesis.kv 385 chan = request.args.get('chan', default = Default_Chan, type = str)
logotron_genesis.kv 386 query = request.args.get('q', default = '', type = str)
logotron_genesis.kv 387
logotron_genesis.kv 388
logotron_genesis.kv 389
logotron_genesis.kv 390 if chan not in Channels:
logotron_genesis.kv 391 return redirect(url_for('log'))
logotron_genesis.kv 392
logotron_genesis.kv 393 nres = 0
logotron_genesis.kv 394 searchres = []
logotron_genesis.kv 395 tokens_orig = []
logotron_genesis.kv 396 search_head = "Query is too short!"
logotron_genesis.kv 397
logotron_genesis.kv 398 if len(query) >= Min_Query_Length:
logotron_genesis.kv 399
logotron_genesis.kv 400 tokens = query.split()
logotron_genesis.kv 401 tokens_standard = []
logotron_genesis.kv 402 from_users = []
logotron_genesis.kv 403
logotron_genesis.kv 404
logotron_genesis.kv 405 for t in tokens:
logotron_genesis.kv 406 if t.startswith("from:") or t.startswith("f:"):
logotron_genesis.kv 407 from_users.append(t.split(':')[1])
logotron_genesis.kv 408 else:
logotron_genesis.kv 409 tokens_standard.append(t)
logotron_genesis.kv 410
logotron_genesis.kv 411 from_users = ['%' + sanitize_speaker(t) + '%' for t in from_users]
logotron_genesis.kv 412 tokens_orig = [re_escape(t) for t in tokens_standard]
logotron_genesis.kv 413 tokens_formed = ['%' + t + '%' for t in tokens_orig]
logotron_genesis.kv 414
logotron_genesis.kv 415
logotron_genesis.kv 416 if from_users == []:
logotron_genesis.kv 417 searchres = query_db(
logotron_genesis.kv 418 '''select * from loglines where chan=%s
logotron_genesis.kv 419 and payload ilike all(%s) order by idx desc limit %s;''',
logotron_genesis.kv 420 [chan,
logotron_genesis.kv 421 tokens_formed,
logotron_genesis.kv 422 Max_Search_Results], one=False)
logotron_genesis.kv 423 else:
logotron_genesis.kv 424 searchres = query_db(
logotron_genesis.kv 425 '''select * from loglines where chan=%s
logotron_genesis.kv 426 and speaker ilike any(%s)
logotron_genesis.kv 427 and payload ilike all(%s) order by idx desc limit %s;''',
logotron_genesis.kv 428 [chan,
logotron_genesis.kv 429 from_users,
logotron_genesis.kv 430 tokens_formed,
logotron_genesis.kv 431 Max_Search_Results], one=False)
logotron_genesis.kv 432
logotron_genesis.kv 433
logotron_genesis.kv 434
logotron_genesis.kv 435 nres = len(searchres)
logotron_genesis.kv 436 search_head = "<b>{0}</b> entries found in {1} for <b>'{2}'</b> :".format(
logotron_genesis.kv 437 nres, chan, html_escape(query))
logotron_genesis.kv 438
logotron_genesis.kv 439
logotron_genesis.kv 440 return render_template('searchres.html',
logotron_genesis.kv 441 query = query,
logotron_genesis.kv 442 nres = nres,
logotron_genesis.kv 443 chan = chan,
logotron_genesis.kv 444 search_head = search_head,
logotron_genesis.kv 445 tokens = tokens_orig,
logotron_genesis.kv 446 loglines = searchres)
logotron_genesis.kv 447
logotron_genesis.kv 448
logotron_genesis.kv 449
logotron_genesis.kv 450 @app.route('/favicon.ico')
logotron_genesis.kv 451 def favicon():
logotron_genesis.kv 452 return redirect(url_for('static', filename='favicon.ico'))
logotron_genesis.kv 453
logotron_genesis.kv 454
logotron_genesis.kv 455
logotron_genesis.kv 456 if __name__ == '__main__':
logotron_genesis.kv 457 app.run(threaded=True, port=WWW_Port)