diff -uNr a/logotron/MANIFEST.TXT b/logotron/MANIFEST.TXT --- a/logotron/MANIFEST.TXT 1b16a235827578071c913387d8fe7601d6d0ab67661b6b27ead056d42cd1e1ca763ff1896f4c02a96551b5b2468801f94ef270176df5008df344c02a1281873f +++ b/logotron/MANIFEST.TXT 145d6f9774ba47abd96e9bea7158cad368632acfa8870d4a1f538ee3bb7aacc2af6ab40c6278863a22b55d7bbba4d4ee0c8e894c5651f2a479cfcb05e820126b @@ -17,3 +17,4 @@ 640165 hide_inactive asciilifeform "Configurable default-hide of idle chans" 641046 search_all_chans asciilifeform "Button to search in all logged chans" 641484 navbar_date_auto asciilifeform "Automatically skip empty days in navbar. Handle 'dawn of time' case." +641773 searchpg_oldlinks asciilifeform "Paginated search; automatic replacement of ancient logger link targets; fix for unicode barf in eat_dump.py." diff -uNr a/logotron/README.txt b/logotron/README.txt --- a/logotron/README.txt 77626555d2a7ac7388c19298a36a8e412b809c6be8532703f18d1e41c197a41a5b3fbc76e964bbd4627b8bb09e3b98cd3ee31bcfb677ea90c1fb3ad9ca8dac4f +++ b/logotron/README.txt 496cd713b60ffcc7f21be04cbbcf6a1316787f72b4e111cc2dd503e20cf73bf66eb739eab8349caa58db3832413f01c17155cd4d64de735a3721610f6205dd78 @@ -159,3 +159,11 @@ (1) Automatically skip empty days in date navigation bar. At the same time, handle empty days correctly when these are requested. (2) Handle 'dawn of time' case for any given channel, to avoid infinite loop insanity from search engines. +#################### +Aug. 2020 Update : +#################### + +(1) Implemented fetch-by-channel-and-index (i.e. /ilog/chan/lineindex links). +(2) Automatically adjust link targets of #ba and 'btcbase' log citations to point to this logger, using the syntax in (1). Original text of link is preserved. Dates are handled correctly. +(3) Search results now properly paginated, with working 'back' and 'forward' buttons. +(4) Fixed ancient unicode bug in eat_dump.py. diff -uNr a/logotron/eat_dump.py b/logotron/eat_dump.py --- a/logotron/eat_dump.py e2818c8381e2a08187a2b3859fe79ac92e2694f14d9214c3fc008e52a48dbe2bb17b40601c30950d3035bbb892b21d2c83d1ab1a45bb595cd1e3ebaf26c97cba +++ b/logotron/eat_dump.py de1cc45eeef929259de9b60c09c535fd3440b09735785aa6225f8ce35ef6fd5cdafddb928991aecc399607ee442370cf08634484bfa70a522785e29350449282 @@ -69,15 +69,17 @@ self_speak = True ## Handle uniturds using the phf algorithm - payload = payload.decode('latin-1') - payload = payload.encode('utf-8') + try: + payload = payload.decode("utf-8") + except UnicodeDecodeError: + payload = payload.decode('latin-1') ## Put in DB: try: exec_db('''insert into loglines (idx, t, chan, era, speaker, self, payload) values (%s, %s, %s, %s, %s, %s, %s) ; ''', [int(idx), datetime.fromtimestamp(time), str(chan), int(era), str(speaker), - bool(self_speak), str(payload)]) + bool(self_speak), payload]) commit_db() except psycopg2.IntegrityError as e: rollback_db() diff -uNr a/logotron/nsabot.conf b/logotron/nsabot.conf --- a/logotron/nsabot.conf 3fb19192427a44dcb19df8673eabef04a84355b0a7bec3a411adfef6b7199969eaf2c02f8536ca89ff5c0332110574c47ff804b323543bedd21353d0ff855d7d +++ b/logotron/nsabot.conf b17807009fa72380c472eeac1c72fbce4a009c541443610520ec2d406530b65ef3c85d4e640d27ac5a61753f4a56f172fd163a224bed5ef2d614e1ff758d04a8 @@ -61,6 +61,9 @@ # Days of inactivity after which chan is hidden by default days_hide = 14 +# Max # of search results per page +max_search = 500 + # On what port will sit the www logtron www_port = 5002 diff -uNr a/logotron/reader.py b/logotron/reader.py --- a/logotron/reader.py a56af711977a55e1332e0a42d8e2acfdfb7c2d9db72dc86c276949c337f0e66acbe6455b19cb156fabb1b242bc7c7f8a69e48987f20d745bcbf44e60d52e9b46 +++ b/logotron/reader.py c9984744c90b16edbb46d64de35f29247076166350a48239295a72270097cb3fd557b38d252b458174361022324d7a0c0714aabc70e68aa375924f62f19497ec @@ -55,6 +55,7 @@ Days_Hide = int(cfg.get("logotron", "days_hide")) # WWW: WWW_Port = int(cfg.get("logotron", "www_port")) + Max_Search_Results = int(cfg.get("logotron", "max_search")) except Exception as e: print "Invalid config: ", e @@ -66,7 +67,6 @@ ### Knobs not made into config yet ### Default_Chan = Channels[0] Min_Query_Length = 3 -Max_Search_Results = 1000 ## Format for Date in Log Lines Date_Short_Format = "%Y-%m-%d" @@ -247,6 +247,12 @@ stdlinks_re = re.compile('(http[^ \[\]]+)') +# For era 1 ('bitcoin-assets') links : +era1_re = re.compile('(\2', payload) - + + # For ancient logs strictly: substitute orig. link with our logger : + if l['era'] < 3: + payload = re.sub(era1_re, + r'/') +def ilog(chan, idx): + # Handle rubbish chan: + if chan not in Channels: + return redirect(url_for('log')) + + # Attempt to locate given chan/idx: + item = query_db( + '''select * from loglines where chan=%s and idx = %s ;''', + [chan, idx], one=True) + + # If given chan/idx not found: + if item == None: + return redirect(url_for('log')) + + # Determine date where item appears in log : + item_date = item['t'].strftime(Date_Short_Format) + + # Go there: + return redirect("/log/" + chan + "/" + item_date + "#" + idx) + + @app.route('/log-raw/') def rawlog(chan): res = "" @@ -467,8 +508,8 @@ # The query params: chan = request.args.get('chan', default = Default_Chan, type = str) query = request.args.get('q', default = '', type = str) - # page_num = request.args.get('page', default = 0, type = int) - + offset = request.args.get('after', default = 0, type = int) + # channels to search in chans = [] @@ -515,32 +556,50 @@ if from_users == []: searchres = query_db( '''select * from loglines where chan = any(%s) - and payload ilike all(%s) order by t desc limit %s;''', + and payload ilike all(%s) order by t desc + limit %s offset %s;''', [(chans,), tokens_formed, - Max_Search_Results], one=False) + Max_Search_Results, + offset], one=False) else: searchres = query_db( '''select * from loglines where chan = any(%s) and speaker ilike any(%s) - and payload ilike all(%s) order by t desc limit %s;''', + and payload ilike all(%s) order by t desc + limit %s offset %s;''', [(chans,), from_users, tokens_formed, - Max_Search_Results], one=False) - - - # Number of entries found + Max_Search_Results, + offset], one=False) + + # Number of search results returned in this query nres = len(searchres) - search_head = "{0} entries found in {1} for '{2}' :".format( - nres, legend, html_escape(query)) - + + # Whether to display 'back' button : + back = (offset != 0) + + # Whether to display 'forward' button : + forw = (nres == Max_Search_Results) + + # Starting index of search results + sres = offset + + # Ending index of search results + eres = offset + min(nres, Max_Search_Results) + # No paging support just yet: return render_template('searchres.html', query = query, - nres = nres, + hquery = html_escape(query), + legend = legend, + sres = sres, + eres = eres, + back = back, + forw = forw, + psize = Max_Search_Results, chan = chan, - search_head = search_head, tokens = tokens_orig, loglines = searchres, showchan = showchan) diff -uNr a/logotron/templates/searchres.html b/logotron/templates/searchres.html --- a/logotron/templates/searchres.html 2b005c8dc669cd7229e8a3cb3932d0b53b7e49fe8d1c7e6a5c33192f81e0937f21324307473a24f7670eb3ff82fb2147dab48b3b8c3e902f25937afc1dca5783 +++ b/logotron/templates/searchres.html edc98bff7fe4f217b1a45a90926ba3abbbd416c50463d9a162886cbff07402f55e8cea5486e212014e2fbb23bace0701f72011452d55a8d12322ffce2ea0e875 @@ -1,12 +1,20 @@ {% extends "layout.html" %} {% block title %} -{{ nres }} Results for {{ query }} in #{{ chan }} +Results {{ sres }} ... {{ eres }} for {{ query }} in {{ legend }} {% endblock %} {% block body %} -
{{ search_head | safe }}
+ +