raw
logotron_genesis.kv     1 #!/usr/bin/python
logotron_genesis.kv     2 
logotron_genesis.kv     3 ##############################################################################
logotron_genesis.kv     4 import ConfigParser, sys
logotron_genesis.kv     5 import psycopg2, psycopg2.extras
logotron_genesis.kv     6 import psycopg2.extensions
logotron_genesis.kv     7 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
logotron_genesis.kv     8 psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
logotron_genesis.kv     9 import time
logotron_genesis.kv    10 import datetime
logotron_genesis.kv    11 from datetime import timedelta
logotron_genesis.kv    12 import sys
logotron_genesis.kv    13 reload(sys)
logotron_genesis.kv    14 sys.setdefaultencoding('utf8')
logotron_genesis.kv    15 import os
logotron_genesis.kv    16 import threading
logotron_genesis.kv    17 import re
logotron_genesis.kv    18 from datetime import datetime
logotron_genesis.kv    19 from urlparse import urljoin
raw_line_export.kv     20 from flask import Flask, request, session, url_for, redirect, Response, \
logotron_genesis.kv    21      render_template, abort, g, flash, _app_ctx_stack, make_response, \
logotron_genesis.kv    22      jsonify
logotron_genesis.kv    23 from flask import Flask
logotron_genesis.kv    24 ##############################################################################
logotron_genesis.kv    25 
logotron_genesis.kv    26 ##############################################################################
logotron_genesis.kv    27 # Single mandatory arg: config file path
hide_inactive.kv       28 
logotron_genesis.kv    29 if len(sys.argv[1:]) != 1:
hide_inactive.kv       30     # Default path for WSGI use (change to yours) :
hide_inactive.kv       31     config_path = "/home/nsabot/logger/nsabot.conf"
hide_inactive.kv       32 else:
hide_inactive.kv       33     # Read Config from given conf file
hide_inactive.kv       34     config_path = sys.argv[1]
logotron_genesis.kv    35 
hide_inactive.kv       36 #config_path = os.path.abspath(config_path)
logotron_genesis.kv    37 cfg = ConfigParser.ConfigParser()
logotron_genesis.kv    38 cfg.readfp(open(config_path))
logotron_genesis.kv    39 
logotron_genesis.kv    40 try:
logotron_genesis.kv    41     # IRCism:
logotron_genesis.kv    42     Nick     = cfg.get("irc", "nick")
logotron_genesis.kv    43     Channels = [x.strip() for x in cfg.get("irc", "chans").split(',')]
logotron_genesis.kv    44     Bots     = [x.strip() for x in cfg.get("logotron", "bots").split(',')]
logotron_genesis.kv    45     Bots.append(Nick) # Add our own bot to the bot list
logotron_genesis.kv    46     # DBism:
logotron_genesis.kv    47     DB_Name  = cfg.get("db", "db_name")
logotron_genesis.kv    48     DB_User  = cfg.get("db", "db_user")
raw_line_export.kv     49     DB_DEBUG = int(cfg.get("db", "db_debug"))
logotron_genesis.kv    50     # Logism:
logotron_genesis.kv    51     Base_URL = cfg.get("logotron", "base_url")
logotron_genesis.kv    52     Era      = int(cfg.get("logotron", "era"))
raw_line_export.kv     53     DEBUG    = int(cfg.get("logotron", "www_dbg"))
raw_line_export.kv     54     Max_Raw_Ln = int(cfg.get("logotron", "max_raw"))
hide_inactive.kv       55     Days_Hide = int(cfg.get("logotron", "days_hide"))
logotron_genesis.kv    56     # WWW:
logotron_genesis.kv    57     WWW_Port = int(cfg.get("logotron", "www_port"))
searchpg_oldlinks.kv   58     Max_Search_Results = int(cfg.get("logotron", "max_search"))
logotron_genesis.kv    59 
logotron_genesis.kv    60 except Exception as e:
logotron_genesis.kv    61     print "Invalid config: ", e
logotron_genesis.kv    62     exit(1)
logotron_genesis.kv    63 
logotron_genesis.kv    64 ##############################################################################
logotron_genesis.kv    65 
logotron_genesis.kv    66 ##############################################################################
logotron_genesis.kv    67 ### Knobs not made into config yet ###
logotron_genesis.kv    68 Default_Chan       = Channels[0]
logotron_genesis.kv    69 Min_Query_Length   = 3
logotron_genesis.kv    70 
logotron_genesis.kv    71 ## Format for Date in Log Lines
logotron_genesis.kv    72 Date_Short_Format  = "%Y-%m-%d"
logotron_genesis.kv    73 ##############################################################################
logotron_genesis.kv    74 
logotron_genesis.kv    75 app = Flask(__name__)
logotron_genesis.kv    76 app.config.from_object(__name__)
logotron_genesis.kv    77 
logotron_genesis.kv    78 def get_db():
logotron_genesis.kv    79     db = getattr(g, 'db', None)
logotron_genesis.kv    80     if db is None:
logotron_genesis.kv    81         db = g.db = psycopg2.connect("dbname=%s user=%s" % (DB_Name, DB_User))
logotron_genesis.kv    82     return db
logotron_genesis.kv    83 
logotron_genesis.kv    84 def close_db():
logotron_genesis.kv    85     if hasattr(g, 'db'):
logotron_genesis.kv    86         g.db.close()
logotron_genesis.kv    87 
logotron_genesis.kv    88 @app.before_request
logotron_genesis.kv    89 def before_request():
logotron_genesis.kv    90     g.db = get_db()
logotron_genesis.kv    91 
logotron_genesis.kv    92 @app.teardown_request
logotron_genesis.kv    93 def teardown_request(exception):
logotron_genesis.kv    94     close_db()
logotron_genesis.kv    95 
logotron_genesis.kv    96 def query_db(query, args=(), one=False):
logotron_genesis.kv    97     cur = get_db().cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logotron_genesis.kv    98     if (DB_DEBUG): print "query: '{0}'".format(query)
logotron_genesis.kv    99     cur.execute(query, args)
logotron_genesis.kv   100     rv = cur.fetchone() if one else cur.fetchall()
logotron_genesis.kv   101     if (DB_DEBUG): print "query res: '{0}'".format(rv)
logotron_genesis.kv   102     return rv
logotron_genesis.kv   103 
logotron_genesis.kv   104 ##############################################################################
logotron_genesis.kv   105 
logotron_genesis.kv   106 ## All eggogs redirect to main page
logotron_genesis.kv   107 @app.errorhandler(404)
logotron_genesis.kv   108 def page_not_found(error):
logotron_genesis.kv   109     return redirect(url_for('log'))
logotron_genesis.kv   110 
logotron_genesis.kv   111 ##############################################################################
logotron_genesis.kv   112 
logotron_genesis.kv   113 html_escape_table = {
logotron_genesis.kv   114     "&": "&",
logotron_genesis.kv   115     '"': """,
logotron_genesis.kv   116     "'": "'",
logotron_genesis.kv   117     ">": ">",
logotron_genesis.kv   118     "<": "&lt;",
logotron_genesis.kv   119 }
logotron_genesis.kv   120 
logotron_genesis.kv   121 def html_escape(text):
logotron_genesis.kv   122     return "".join(html_escape_table.get(c,c) for c in text)
logotron_genesis.kv   123 
logotron_genesis.kv   124 ##############################################################################
logotron_genesis.kv   125 
logotron_genesis.kv   126 ## Get base URL
logotron_genesis.kv   127 def get_base():
logotron_genesis.kv   128     if DEBUG:
logotron_genesis.kv   129         return request.host_url
logotron_genesis.kv   130     return Base_URL
logotron_genesis.kv   131 
logotron_genesis.kv   132 
logotron_genesis.kv   133 # Get perma-URL corresponding to given log line
logotron_genesis.kv   134 def line_url(l):
logotron_genesis.kv   135     return "{0}log/{1}/{2}#{3}".format(get_base(),
logotron_genesis.kv   136                                        l['chan'],
logotron_genesis.kv   137                                        l['t'].strftime(Date_Short_Format),
logotron_genesis.kv   138                                        l['idx'])
logotron_genesis.kv   139 
hide_inactive.kv      140 def gen_chanlist(selected_chan, show_all_chans=False):
logotron_genesis.kv   141     # Get current time
logotron_genesis.kv   142     now = datetime.now()
hide_inactive.kv      143     # Data for channel display :
hide_inactive.kv      144     chan_tbl = {}
logotron_genesis.kv   145     for chan in Channels:
hide_inactive.kv      146         chan_tbl[chan] = {}
hide_inactive.kv      147         chan_tbl[chan]['show'] = False
hide_inactive.kv      148         
logotron_genesis.kv   149         chan_formed = chan
logotron_genesis.kv   150         if chan == selected_chan:
logotron_genesis.kv   151             chan_formed = "<span class='highlight'>" + chan + "</span>"
hide_inactive.kv      152         
hide_inactive.kv      153         chan_tbl[chan]['link'] = """<a href="{0}log/{1}"><b>{2}</b></a>""".format(
logotron_genesis.kv   154             get_base(), chan, chan_formed)
hide_inactive.kv      155         
logotron_genesis.kv   156         last_time = query_db(
logotron_genesis.kv   157             '''select t, idx from loglines where chan=%s
logotron_genesis.kv   158             and idx = (select max(idx) from loglines where chan=%s) ;''',
logotron_genesis.kv   159             [chan, chan], one=True)
logotron_genesis.kv   160 
logotron_genesis.kv   161         last_time_txt = ""
hide_inactive.kv      162         time_field = ""
logotron_genesis.kv   163         if last_time != None:
logotron_genesis.kv   164             span = (now - last_time['t'])
logotron_genesis.kv   165             days = span.days
logotron_genesis.kv   166             hours = span.seconds/3600
logotron_genesis.kv   167             minutes = (span.seconds%3600)/60
logotron_genesis.kv   168 
logotron_genesis.kv   169             if days != 0:
logotron_genesis.kv   170                 last_time_txt += '%dd ' % days
logotron_genesis.kv   171             if hours != 0:
logotron_genesis.kv   172                 last_time_txt += '%dh ' % hours
logotron_genesis.kv   173             if minutes != 0:
logotron_genesis.kv   174                 last_time_txt += '%dm' % minutes
hide_inactive.kv      175             
hide_inactive.kv      176             time_field = """<i><a href="{0}log/{1}/{2}#{3}">{4}</a></i>""".format(
logotron_genesis.kv   177                 get_base(),
logotron_genesis.kv   178                 chan,
logotron_genesis.kv   179                 last_time['t'].strftime(Date_Short_Format),
logotron_genesis.kv   180                 last_time['idx'],
logotron_genesis.kv   181                 last_time_txt)
logotron_genesis.kv   182 
hide_inactive.kv      183             if (days <= Days_Hide) or (chan == selected_chan) or show_all_chans:
hide_inactive.kv      184                 chan_tbl[chan]['show'] = True
logotron_genesis.kv   185 
hide_inactive.kv      186         chan_tbl[chan]['time'] = time_field
hide_inactive.kv      187     
hide_inactive.kv      188     ## Generate channel selector bar :
hide_inactive.kv      189     s = """<table align="center" class="chantable"><tr>"""
hide_inactive.kv      190     for chan in Channels:
hide_inactive.kv      191         if chan_tbl[chan]['show']:
hide_inactive.kv      192             s += """<th>{0}</th>""".format(chan_tbl[chan]['link'])
hide_inactive.kv      193     s += "</tr><tr>"
hide_inactive.kv      194     ## Generate last-activ. links for above :
hide_inactive.kv      195     for chan in Channels:
hide_inactive.kv      196         if chan_tbl[chan]['show']:
hide_inactive.kv      197             s += """<td>{0}</td>""".format(chan_tbl[chan]['time'])
hide_inactive.kv      198     # wrap up:
search_all_chans.kv   199     s += "</tr></table>"
logotron_genesis.kv   200     return s
logotron_genesis.kv   201 
logotron_genesis.kv   202 
logotron_genesis.kv   203 # Make above callable from inside htm templater:
logotron_genesis.kv   204 app.jinja_env.globals.update(gen_chanlist=gen_chanlist)
logotron_genesis.kv   205 
logotron_genesis.kv   206 
logotron_genesis.kv   207 # HTML Tag Regex
logotron_genesis.kv   208 tag_regex = re.compile("(<[^>]+>)")
logotron_genesis.kv   209 
logotron_genesis.kv   210 
logotron_genesis.kv   211 # Find the segments of a block of text which constitute HTML tags
logotron_genesis.kv   212 def get_link_intervals(str):
logotron_genesis.kv   213     links = []
logotron_genesis.kv   214     span = []
logotron_genesis.kv   215     for match in tag_regex.finditer(str):
logotron_genesis.kv   216         span = match.span()
logotron_genesis.kv   217         links += [span]
logotron_genesis.kv   218     return links
logotron_genesis.kv   219 
logotron_genesis.kv   220 
logotron_genesis.kv   221 # Highlight all matched tokens in given text
logotron_genesis.kv   222 def highlight_matches(strings, text):
logotron_genesis.kv   223     e = '(' + ('|'.join(strings)) + ')'
logotron_genesis.kv   224     return re.sub(e,
logotron_genesis.kv   225                   r"""<span class='highlight'>\1</span>""",
logotron_genesis.kv   226                   text,
logotron_genesis.kv   227                   flags=re.I)
logotron_genesis.kv   228 
logotron_genesis.kv   229 
logotron_genesis.kv   230 # Highlight matched tokens in the display of a search result logline,
logotron_genesis.kv   231 # but leave HTML tags alone
logotron_genesis.kv   232 def highlight_text(strings, text):
logotron_genesis.kv   233     result = ""
logotron_genesis.kv   234     last = 0
logotron_genesis.kv   235     for i in get_link_intervals(text):
logotron_genesis.kv   236         i_start, i_end = i
logotron_genesis.kv   237         result += highlight_matches(strings, text[last:i_start])
logotron_genesis.kv   238         result += text[i_start:i_end] # the HTML tag, leave it alone
logotron_genesis.kv   239         last = i_end
logotron_genesis.kv   240     result += highlight_matches(strings, text[last:]) # last block
logotron_genesis.kv   241     return result
logotron_genesis.kv   242 
logotron_genesis.kv   243 
logotron_genesis.kv   244 # Regexps used in format_logline:
uniturds_etc.kv       245 boxlinks_re = re.compile(
uniturds_etc.kv       246     '\[\s*<a href="(http[^ \[\]]+)"[^>]*>[^ <]+</a>\s*\]\[([^\[\]]+)\]')
uniturds_etc.kv       247 
logotron_genesis.kv   248 stdlinks_re = re.compile('(http[^ \[\]]+)')
logotron_genesis.kv   249 
searchpg_oldlinks.kv  250 # For era 1 ('bitcoin-assets') links :
searchpg_oldlinks.kv  251 era1_re = re.compile('(<a href="http[^ \[\]]\/\/log\d*\.bitcoin-assets\.com\/\?date=\d+-\d+-\d+#)(\d+)')
searchpg_oldlinks.kv  252 
searchpg_oldlinks.kv  253 # For era 2 ('btcbase') links :
searchpg_oldlinks.kv  254 era2_re = re.compile('(<a href="http[^ \[\]]\/\/btcbase\.org\/log\/\d+-\d+-\d+#)(\d+)')
searchpg_oldlinks.kv  255 
logotron_genesis.kv   256 
logotron_genesis.kv   257 ## Format given log line for display
search_all_chans.kv   258 def format_logline(l, highlights = [], select=[], showchan=False):
logotron_genesis.kv   259     payload = html_escape(l['payload'])
logotron_genesis.kv   260     
logotron_genesis.kv   261     # Format ordinary links:
uniturds_etc.kv       262     payload = re.sub(stdlinks_re,
uniturds_etc.kv       263                      r'<a href="\1" target=\'_blank\'>\1</a>', payload)
logotron_genesis.kv   264 
logotron_genesis.kv   265     # Now also format [link][text] links :
uniturds_etc.kv       266     payload = re.sub(boxlinks_re,
uniturds_etc.kv       267                      r'<a href="\1"  target=\'_blank\'>\2</a>', payload)
searchpg_oldlinks.kv  268     
searchpg_oldlinks.kv  269     # For ancient logs strictly: substitute orig. link with our logger :
searchpg_oldlinks.kv  270     if l['era'] < 3:
searchpg_oldlinks.kv  271         payload = re.sub(era1_re,
searchpg_oldlinks.kv  272                          r'<a href="/ilog/{0}/\2'.format(l['chan']),
searchpg_oldlinks.kv  273                          payload)
searchpg_oldlinks.kv  274     
searchpg_oldlinks.kv  275     # Adjust era 2 links in all cases:
searchpg_oldlinks.kv  276     payload = re.sub(era2_re,
searchpg_oldlinks.kv  277                      r'<a href="/ilog/{0}/\2'.format(l['chan']),
searchpg_oldlinks.kv  278                      payload)
searchpg_oldlinks.kv  279     
logotron_genesis.kv   280     # If this is a search result, illuminate the matched strings:
logotron_genesis.kv   281     if highlights != []:
logotron_genesis.kv   282         payload = highlight_text(highlights, payload)
logotron_genesis.kv   283     
logotron_genesis.kv   284     bot = ""
logotron_genesis.kv   285     if l['speaker'] in Bots:
logotron_genesis.kv   286         bot = " bot"
logotron_genesis.kv   287 
multsel_and_datef...  288     # default -- no selection
multsel_and_datef...  289     dclass = l['speaker']
multsel_and_datef...  290 
multsel_and_datef...  291     # If selection is given:
multsel_and_datef...  292     if select != []:
multsel_and_datef...  293         ss, se = select
multsel_and_datef...  294         if ss <= l['idx'] <= se:
multsel_and_datef...  295             dclass = "highlight"
multsel_and_datef...  296 
sept_fixes.kv         297     speaker = l['speaker']
sept_fixes.kv         298     separator = ":"
sept_fixes.kv         299 
search_all_chans.kv   300     if showchan:
search_all_chans.kv   301         speaker = '<small>(' + l['chan'] + ')</small> ' + speaker
search_all_chans.kv   302     
sept_fixes.kv         303     # If 'action', annotate:
sept_fixes.kv         304     if l['self']:
sept_fixes.kv         305         separator = ""
sept_fixes.kv         306         payload = "<i>" + payload + "</i>"
sept_fixes.kv         307         speaker = "<i>" + speaker + "</i>"
sept_fixes.kv         308     
logotron_genesis.kv   309     # HTMLize the given line :
multsel_and_datef...  310     s = ("<div id='{0}' class='{6}{5}'>"
logotron_genesis.kv   311          "<a class='nick' title='{2}'"
sept_errata.kv        312          " href=\"{3}\">{1}</a>{7} {4}</div>").format(l['idx'],
sept_fixes.kv         313                                                       speaker,
sept_fixes.kv         314                                                       l['t'],
sept_fixes.kv         315                                                       line_url(l),
sept_fixes.kv         316                                                       payload,
sept_fixes.kv         317                                                       bot,
sept_fixes.kv         318                                                       dclass,
sept_fixes.kv         319                                                       separator)
logotron_genesis.kv   320     return s
logotron_genesis.kv   321 
logotron_genesis.kv   322 # Make above callable from inside htm templater:
logotron_genesis.kv   323 app.jinja_env.globals.update(format_logline=format_logline) 
logotron_genesis.kv   324 
logotron_genesis.kv   325 
uniturds_etc.kv       326 @app.route('/rnd/<chan>')
uniturds_etc.kv       327 def rnd(chan):
uniturds_etc.kv       328     # Handle rubbish chan:
uniturds_etc.kv       329     if chan not in Channels:
uniturds_etc.kv       330         return redirect(url_for('log'))
uniturds_etc.kv       331 
uniturds_etc.kv       332     rnd_line = query_db(
uniturds_etc.kv       333         '''select * from loglines where chan=%s
uniturds_etc.kv       334         order by random() limit 1 ;''',
uniturds_etc.kv       335         [chan], one=True)
uniturds_etc.kv       336 
uniturds_etc.kv       337     return redirect(line_url(rnd_line))
uniturds_etc.kv       338 
uniturds_etc.kv       339 
logotron_genesis.kv   340 @app.route('/log/<chan>/<date>')
logotron_genesis.kv   341 @app.route('/log/<chan>', defaults={'date': None})
logotron_genesis.kv   342 @app.route('/log/', defaults={'chan': Default_Chan, 'date': None})
logotron_genesis.kv   343 @app.route('/log', defaults={'chan': Default_Chan, 'date': None})
logotron_genesis.kv   344 def log(chan, date):
logotron_genesis.kv   345     # Handle rubbish chan:
logotron_genesis.kv   346     if chan not in Channels:
logotron_genesis.kv   347         return redirect(url_for('log'))
logotron_genesis.kv   348 
multsel_and_datef...  349     # Get possible selection start and end
multsel_and_datef...  350     sel_start = request.args.get('ss', default = 0, type = int)
multsel_and_datef...  351     sel_end = request.args.get('se', default = 0, type = int)
uniturds_etc.kv       352 
uniturds_etc.kv       353     # Get possible 'reverse gear'
uniturds_etc.kv       354     rev = request.args.get('rev', default = 0, type = int)
hide_inactive.kv      355 
hide_inactive.kv      356     # Get possible 'show all'
hide_inactive.kv      357     show_all = request.args.get('all', default = 0, type = int)
multsel_and_datef...  358     
logotron_genesis.kv   359     # Get current time
logotron_genesis.kv   360     now = datetime.now()
logotron_genesis.kv   361 
logotron_genesis.kv   362     # Whether we are viewing 'current' tail
logotron_genesis.kv   363     tail = False
logotron_genesis.kv   364     
logotron_genesis.kv   365     # If viewing 'current' log:
logotron_genesis.kv   366     if date == None:
logotron_genesis.kv   367         date = now.strftime(Date_Short_Format)
logotron_genesis.kv   368         tail = True
logotron_genesis.kv   369 
logotron_genesis.kv   370     # Parse given date, and redirect to default log if rubbish:
logotron_genesis.kv   371     try:
logotron_genesis.kv   372         day_start = datetime.strptime(date, Date_Short_Format)
logotron_genesis.kv   373     except Exception, e:
logotron_genesis.kv   374         return redirect(url_for('log'))
logotron_genesis.kv   375     
logotron_genesis.kv   376     # Determine the end of the interval being shown
logotron_genesis.kv   377     day_end = day_start + timedelta(days=1)
logotron_genesis.kv   378 
multsel_and_datef...  379     # Enable 'tail' is day_end is after end of current day
rle_errata.kv         380     if day_end > now:
multsel_and_datef...  381         tail = True
navbar_date_auto.kv   382 
logotron_genesis.kv   383     # Get the loglines from DB
logotron_genesis.kv   384     lines = query_db(
logotron_genesis.kv   385         '''select * from loglines where chan=%s
logotron_genesis.kv   386         and t between %s and %s order by idx asc;''',
logotron_genesis.kv   387                      [chan, day_start, day_end], one=False)
logotron_genesis.kv   388     
uniturds_etc.kv       389     # Optional 'reverse gear' knob:
uniturds_etc.kv       390     if rev == 1:
uniturds_etc.kv       391         lines.reverse()
navbar_date_auto.kv   392     
navbar_date_auto.kv   393     # Generate navbar for the given date:
navbar_date_auto.kv   394     prev_day = ""
navbar_date_auto.kv   395     next_day = ""
navbar_date_auto.kv   396     
navbar_date_auto.kv   397     prev_t = query_db(
navbar_date_auto.kv   398         '''select t from loglines where chan=%s
navbar_date_auto.kv   399         and t < %s order by idx desc limit 1;''',
navbar_date_auto.kv   400         [chan, day_start], one=True)
navbar_date_auto.kv   401     
navbar_date_auto.kv   402     if prev_t != None:
navbar_date_auto.kv   403         prev_day = prev_t['t'].strftime(Date_Short_Format)
navbar_date_auto.kv   404     
navbar_date_auto.kv   405     if not tail:
navbar_date_auto.kv   406         next_t = query_db(
navbar_date_auto.kv   407             '''select t from loglines where chan=%s
navbar_date_auto.kv   408             and t > %s order by idx asc limit 1;''',
navbar_date_auto.kv   409             [chan, day_end], one=True)
navbar_date_auto.kv   410         
navbar_date_auto.kv   411         if next_t != None:
navbar_date_auto.kv   412             next_day = next_t['t'].strftime(Date_Short_Format)
navbar_date_auto.kv   413     
logotron_genesis.kv   414     # Return the HTMLized text
logotron_genesis.kv   415     return render_template('log.html',
logotron_genesis.kv   416                            chan     = chan,
logotron_genesis.kv   417                            loglines = lines,
multsel_and_datef...  418                            sel      = (sel_start, sel_end),
logotron_genesis.kv   419                            date     = date,
navbar_date_auto.kv   420                            prev_day = prev_day,
navbar_date_auto.kv   421                            next_day = next_day,
hide_inactive.kv      422                            rev      = not rev,
hide_inactive.kv      423                            show_all = show_all,
hide_inactive.kv      424                            idle_day = Days_Hide)
logotron_genesis.kv   425 
logotron_genesis.kv   426 
searchpg_oldlinks.kv  427 # Primarily for use with 'era 1' and 'era 2' :
searchpg_oldlinks.kv  428 # Get arbitrary log item by chan and raw line index
searchpg_oldlinks.kv  429 @app.route('/ilog/<chan>/<idx>')
searchpg_oldlinks.kv  430 def ilog(chan, idx):
searchpg_oldlinks.kv  431     # Handle rubbish chan:
searchpg_oldlinks.kv  432     if chan not in Channels:
searchpg_oldlinks.kv  433         return redirect(url_for('log'))
searchpg_oldlinks.kv  434 
searchpg_oldlinks.kv  435     # Attempt to locate given chan/idx:
searchpg_oldlinks.kv  436     item = query_db(
searchpg_oldlinks.kv  437         '''select * from loglines where chan=%s and idx = %s ;''',
searchpg_oldlinks.kv  438         [chan, idx], one=True)
searchpg_oldlinks.kv  439 
searchpg_oldlinks.kv  440     # If given chan/idx not found:
searchpg_oldlinks.kv  441     if item == None:
searchpg_oldlinks.kv  442         return redirect(url_for('log'))
searchpg_oldlinks.kv  443     
searchpg_oldlinks.kv  444     # Determine date where item appears in log :
searchpg_oldlinks.kv  445     item_date = item['t'].strftime(Date_Short_Format)
searchpg_oldlinks.kv  446     
searchpg_oldlinks.kv  447     # Go there:
searchpg_oldlinks.kv  448     return redirect("/log/" + chan + "/" + item_date + "#" + idx)
searchpg_oldlinks.kv  449     
searchpg_oldlinks.kv  450 
raw_line_export.kv    451 @app.route('/log-raw/<chan>')
raw_line_export.kv    452 def rawlog(chan):
raw_line_export.kv    453     res = ""
raw_line_export.kv    454     
raw_line_export.kv    455     # Handle rubbish chan:
raw_line_export.kv    456     if chan not in Channels:
raw_line_export.kv    457         return Response("EGGOG: No such Channel!", mimetype='text/plain')
raw_line_export.kv    458     
raw_line_export.kv    459     # Get start and end indices:
raw_line_export.kv    460     idx_start = request.args.get('istart', default = 0, type = int)
raw_line_export.kv    461     idx_end = request.args.get('iend', default = 0, type = int)
raw_line_export.kv    462     
raw_line_export.kv    463     # Malformed bounds?
raw_line_export.kv    464     if idx_start > idx_end:
raw_line_export.kv    465         return Response("EGGOG: Start must precede End!",
raw_line_export.kv    466                         mimetype='text/plain')
raw_line_export.kv    467 
raw_line_export.kv    468     # Demanded too many in one burst ?
raw_line_export.kv    469     if (idx_end - idx_start) > Max_Raw_Ln :
raw_line_export.kv    470         return Response("EGGOG: May request Max. of %s Lines !" % Max_Raw_Ln,
raw_line_export.kv    471                         mimetype='text/plain')
raw_line_export.kv    472 
raw_line_export.kv    473     # Get the loglines from DB
raw_line_export.kv    474     lines = query_db(
raw_line_export.kv    475         '''select * from loglines where chan=%s
raw_line_export.kv    476         and idx between %s and %s order by idx asc;''',
raw_line_export.kv    477                      [chan, idx_start, idx_end], one=False)
raw_line_export.kv    478     
raw_line_export.kv    479     # Retrieve raw lines in classical Phf format:
raw_line_export.kv    480     for l in lines:
raw_line_export.kv    481         action = ""
raw_line_fix.kv       482         speaker = "%s;" % l['speaker']
raw_line_export.kv    483         if l['self']:
raw_line_export.kv    484             action = "*;"
raw_line_fix.kv       485             speaker = "%s " % l['speaker']
raw_line_fix.kv       486         res += "%s;%s;%s%s%s\n" % (l['idx'],
raw_line_export.kv    487                                     l['t'].strftime('%s'),
raw_line_export.kv    488                                     action,
raw_line_fix.kv       489                                     speaker,
raw_line_export.kv    490                                     l['payload'])
raw_line_export.kv    491     
raw_line_export.kv    492     # Return plain text:
raw_line_export.kv    493     return Response(res, mimetype='text/plain')
raw_line_export.kv    494 
logotron_genesis.kv   495 
logotron_genesis.kv   496 Name_Chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"
logotron_genesis.kv   497 
logotron_genesis.kv   498 def sanitize_speaker(s):
logotron_genesis.kv   499     return "".join([ch for ch in s if ch in Name_Chars])
logotron_genesis.kv   500 
logotron_genesis.kv   501 
logotron_genesis.kv   502 def re_escape(s):
logotron_genesis.kv   503     return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s)
logotron_genesis.kv   504 
logotron_genesis.kv   505 # Search knob. Supports 'chan' parameter.
logotron_genesis.kv   506 @app.route('/log-search')
logotron_genesis.kv   507 def logsearch():
logotron_genesis.kv   508     # The query params:
logotron_genesis.kv   509     chan = request.args.get('chan', default = Default_Chan, type = str)
logotron_genesis.kv   510     query = request.args.get('q', default = '', type = str)
searchpg_oldlinks.kv  511     offset = request.args.get('after', default = 0, type = int)
searchpg_oldlinks.kv  512     
search_all_chans.kv   513     # channels to search in
search_all_chans.kv   514     chans = []
logotron_genesis.kv   515 
search_all_chans.kv   516     # whether to indicate chan per log line
search_all_chans.kv   517     showchan = False
search_all_chans.kv   518 
search_all_chans.kv   519     if chan == 'all':
search_all_chans.kv   520         # search in all logged channels
search_all_chans.kv   521         chans = Channels
search_all_chans.kv   522         legend = "<i>all logged channels</i>"
search_all_chans.kv   523         showchan = True
search_all_chans.kv   524     else:
search_all_chans.kv   525         # Handle possible rubbish chan:
search_all_chans.kv   526         if chan not in Channels:
search_all_chans.kv   527             return redirect(url_for('log'))
search_all_chans.kv   528         else:
search_all_chans.kv   529             # search in selected channel only
search_all_chans.kv   530             chans = [chan]
search_all_chans.kv   531             legend = chan
search_all_chans.kv   532     
logotron_genesis.kv   533     nres = 0
logotron_genesis.kv   534     searchres   = []
logotron_genesis.kv   535     tokens_orig = []
logotron_genesis.kv   536     search_head = "Query is too short!"
logotron_genesis.kv   537     # Forbid query that is too short:
logotron_genesis.kv   538     if len(query) >= Min_Query_Length:
logotron_genesis.kv   539         # Get the search tokens to use:
shlex_removal.kv      540         tokens = query.split()
logotron_genesis.kv   541         tokens_standard = []
logotron_genesis.kv   542         from_users = []
logotron_genesis.kv   543 
logotron_genesis.kv   544         # separate out "from:foo" tokens and ordinary:
logotron_genesis.kv   545         for t in tokens:
logotron_genesis.kv   546             if t.startswith("from:") or t.startswith("f:"):
logotron_genesis.kv   547                 from_users.append(t.split(':')[1]) # Record user for 'from' query
logotron_genesis.kv   548             else:
logotron_genesis.kv   549                 tokens_standard.append(t)
logotron_genesis.kv   550         
logotron_genesis.kv   551         from_users = ['%' + sanitize_speaker(t) + '%' for t in from_users]
logotron_genesis.kv   552         tokens_orig = [re_escape(t) for t in tokens_standard]
logotron_genesis.kv   553         tokens_formed = ['%' + t + '%' for t in tokens_orig]
logotron_genesis.kv   554         
logotron_genesis.kv   555         # Query is usable; perform the search on DB and get the finds
logotron_genesis.kv   556         if from_users == []:
logotron_genesis.kv   557             searchres = query_db(
search_all_chans.kv   558                 '''select * from loglines where chan = any(%s)
searchpg_oldlinks.kv  559                 and payload ilike all(%s) order by t desc
searchpg_oldlinks.kv  560                 limit %s offset %s;''',
search_all_chans.kv   561                 [(chans,),
logotron_genesis.kv   562                  tokens_formed,
searchpg_oldlinks.kv  563                  Max_Search_Results,
searchpg_oldlinks.kv  564                  offset], one=False)
logotron_genesis.kv   565         else:
logotron_genesis.kv   566             searchres = query_db(
search_all_chans.kv   567                 '''select * from loglines where chan = any(%s)
logotron_genesis.kv   568                 and speaker ilike any(%s)
searchpg_oldlinks.kv  569                 and payload ilike all(%s) order by t desc
searchpg_oldlinks.kv  570                 limit %s offset %s;''',
search_all_chans.kv   571                 [(chans,),
logotron_genesis.kv   572                  from_users,
logotron_genesis.kv   573                  tokens_formed,
searchpg_oldlinks.kv  574                  Max_Search_Results,
searchpg_oldlinks.kv  575                  offset], one=False)
searchpg_oldlinks.kv  576         
searchpg_oldlinks.kv  577         # Number of search results returned in this query
logotron_genesis.kv   578         nres = len(searchres)
searchpg_oldlinks.kv  579 
searchpg_oldlinks.kv  580         # Whether to display 'back' button :
searchpg_oldlinks.kv  581         back = (offset != 0)
searchpg_oldlinks.kv  582         
searchpg_oldlinks.kv  583         # Whether to display 'forward' button :
searchpg_oldlinks.kv  584         forw = (nres == Max_Search_Results)
searchpg_oldlinks.kv  585         
searchpg_oldlinks.kv  586         # Starting index of search results
searchpg_oldlinks.kv  587         sres = offset
searchpg_oldlinks.kv  588         
searchpg_oldlinks.kv  589         # Ending index of search results
searchpg_oldlinks.kv  590         eres = offset + min(nres, Max_Search_Results)
searchpg_oldlinks.kv  591         
logotron_genesis.kv   592     # No paging support just yet:
logotron_genesis.kv   593     return render_template('searchres.html',
logotron_genesis.kv   594                            query       = query,
searchpg_oldlinks.kv  595                            hquery      = html_escape(query),
searchpg_oldlinks.kv  596                            legend      = legend,
searchpg_oldlinks.kv  597                            sres        = sres,
searchpg_oldlinks.kv  598                            eres        = eres,
searchpg_oldlinks.kv  599                            back        = back,
searchpg_oldlinks.kv  600                            forw        = forw,
searchpg_oldlinks.kv  601                            psize       = Max_Search_Results,
logotron_genesis.kv   602                            chan        = chan,
logotron_genesis.kv   603                            tokens      = tokens_orig,
search_all_chans.kv   604                            loglines    = searchres,
search_all_chans.kv   605                            showchan    = showchan)
logotron_genesis.kv   606 
logotron_genesis.kv   607 # Comment this out if you don't have one
logotron_genesis.kv   608 @app.route('/favicon.ico')
logotron_genesis.kv   609 def favicon():
logotron_genesis.kv   610     return redirect(url_for('static', filename='favicon.ico'))
logotron_genesis.kv   611 
logotron_genesis.kv   612 
logotron_genesis.kv   613 ## App Mode
logotron_genesis.kv   614 if __name__ == '__main__':
logotron_genesis.kv   615     app.run(threaded=True, port=WWW_Port)