9979-presence 1 import calendar
9979-presence 2
9985-single-thread 3 from peer import Peer
9982-getdata 4 from message import EMPTY_CHAIN
9979-presence 5 from message import Message
genesis 6 import sqlite3
genesis 7 import imp
9982-getdata 8 import binascii
9987-embargoing 9 import logging
9984-unbork-at-co... 10 import datetime
9979-presence 11 import time
9982-getdata 12 import caribou
9982-getdata 13
genesis 14 from itertools import chain
genesis 15
9979-presence 16 KNOBS=(
9979-presence 17 {
9979-presence 18 'max_bounces': 3,
9982-getdata 19 'embargo_interval_seconds': 1,
9982-getdata 20 'rubbish_interval_seconds': 10,
9982-getdata 21 'nick': '',
9982-getdata 22 'order_buffer_check_seconds': 5 * 60,
9982-getdata 23 'order_buffer_expiration_seconds': 5 * 60,
9982-getdata 24 'short_buffer_expiration_seconds': 1,
9979-presence 25 'short_buffer_check_interval_seconds': 1,
9979-presence 26 'peer_offline_interval_seconds': 60,
9979-presence 27 'peer_away_interval_seconds': 10 * 60,
9979-presence 28 'presence_check_seconds': 5,
9979-presence 29 }
9979-presence 30 )
9983-knobs 31
genesis 32 class State(object):
9982-getdata 33 def __init__(self, station, db_path=None):
9982-getdata 34 self.station = station
9982-getdata 35 if db_path:
9982-getdata 36 self.conn = sqlite3.connect(db_path)
9982-getdata 37 else:
9982-getdata 38 self.conn = sqlite3.connect("file::memory:")
9982-getdata 39
9982-getdata 40 cursor = self.cursor()
9982-getdata 41 cursor.execute("create table if not exists handle_self_chain(id integer primary key autoincrement,\
9982-getdata 42 handle string not null,\
9982-getdata 43 message_hash blob not null)")
9982-getdata 44
9982-getdata 45 cursor.execute("create table if not exists broadcast_self_chain(id integer primary key autoincrement,\
9982-getdata 46 message_hash blob not null)")
9982-getdata 47
9982-getdata 48 cursor.execute("create table if not exists net_chain(id integer primary key autoincrement,\
9982-getdata 49 message_hash blob not null)")
9982-getdata 50
9982-getdata 51 cursor.execute("create table if not exists at(handle_id integer,\
9982-getdata 52 address text not null,\
9982-getdata 53 port integer not null,\
9982-getdata 54 updated_at datetime default null,\
9982-getdata 55 unique(handle_id, address, port))")
9982-getdata 56
9982-getdata 57 cursor.execute("create table if not exists wot(peer_id integer primary key autoincrement)")
9982-getdata 58
9982-getdata 59 cursor.execute("create table if not exists handles(handle_id integer primary key,\
9982-getdata 60 peer_id integer,\
9982-getdata 61 handle text,\
9982-getdata 62 unique(handle))")
9982-getdata 63
9982-getdata 64 cursor.execute("create table if not exists keys(peer_id intenger,\
9982-getdata 65 key text,\
9982-getdata 66 used_at datetime default current_timestamp,\
9982-getdata 67 unique(key))")
9982-getdata 68
9982-getdata 69 cursor.execute("create table if not exists log(\
9982-getdata 70 message_bytes blob not null,\
9982-getdata 71 message_hash text not null, \
9982-getdata 72 command integer not null, \
9982-getdata 73 timestamp datetime not null, \
9982-getdata 74 created_at datetime default current_timestamp)")
9982-getdata 75
9982-getdata 76 cursor.execute("create table if not exists knobs(\
9982-getdata 77 name text not null,\
9982-getdata 78 value text not null)")
9982-getdata 79
9982-getdata 80
9982-getdata 81 if db_path:
9982-getdata 82 caribou.upgrade(db_path, "migrations")
9982-getdata 83
9982-getdata 84 self.conn.commit()
9988-hash-dedup 85
9985-single-thread 86 def cursor(self):
9985-single-thread 87 return self.conn.cursor()
9985-single-thread 88
9982-getdata 89 def update_handle_self_chain(self, handle, message_hash):
9982-getdata 90 cursor = self.cursor()
9982-getdata 91 cursor.execute("insert into handle_self_chain(handle, message_hash) values(?, ?)", (handle, buffer(message_hash)))
9982-getdata 92 self.conn.commit()
9982-getdata 93
9982-getdata 94 def get_handle_self_chain(self, handle):
9982-getdata 95 cursor = self.cursor()
9982-getdata 96 results = cursor.execute("select message_hash from handle_self_chain where handle=?\
9982-getdata 97 order by id desc limit 1", (handle,)).fetchone()
9982-getdata 98 if results is not None:
9982-getdata 99 return results[0][:]
9982-getdata 100 else:
9982-getdata 101 return EMPTY_CHAIN
9982-getdata 102
9982-getdata 103 def update_broadcast_self_chain(self, message_hash):
9982-getdata 104 cursor = self.cursor()
9982-getdata 105 cursor.execute("insert into broadcast_self_chain(message_hash) values(?)", (buffer(message_hash),))
9982-getdata 106 self.conn.commit()
9982-getdata 107
9982-getdata 108 def get_broadcast_self_chain(self):
9982-getdata 109 cursor = self.cursor()
9982-getdata 110 results = cursor.execute("select message_hash from broadcast_self_chain order by id desc limit 1").fetchone()
9982-getdata 111 if results is not None:
9982-getdata 112 return results[0][:]
9982-getdata 113 else:
9982-getdata 114 return EMPTY_CHAIN
9982-getdata 115
9982-getdata 116 def update_net_chain(self, message_hash):
9982-getdata 117 self.cursor().execute("insert into net_chain(message_hash) values(?)", (buffer(message_hash),))
9982-getdata 118 self.conn.commit()
9982-getdata 119
9982-getdata 120 def get_net_chain(self):
9982-getdata 121 cursor = self.cursor()
9982-getdata 122 results = cursor.execute("select message_hash from net_chain order by id desc limit 1").fetchone()
9982-getdata 123 if results is not None:
9982-getdata 124 return results[0][:]
9982-getdata 125 else:
9982-getdata 126 return EMPTY_CHAIN
9982-getdata 127
9983-knobs 128 def get_knobs(self):
9983-knobs 129 cursor = self.cursor()
9983-knobs 130 results = cursor.execute("select name, value from knobs order by name asc").fetchall()
9983-knobs 131 knobs = {}
9983-knobs 132 for result in results:
9983-knobs 133 knobs[result[0]] = result[1]
9983-knobs 134 for key in KNOBS.keys():
9983-knobs 135 if not knobs.get(key):
9983-knobs 136 knobs[key] = KNOBS[key]
9983-knobs 137 return knobs
9983-knobs 138
9983-knobs 139 def get_knob(self, knob_name):
9983-knobs 140 cursor = self.cursor()
9983-knobs 141 result = cursor.execute("select value from knobs where name=?", (knob_name,)).fetchone()
9983-knobs 142 if result:
9983-knobs 143 return result[0]
9983-knobs 144 elif KNOBS.get(knob_name):
9983-knobs 145 return KNOBS.get(knob_name)
9983-knobs 146 else:
9983-knobs 147 return None
9983-knobs 148
9983-knobs 149 def set_knob(self, knob_name, knob_value):
9983-knobs 150 cursor = self.cursor()
9983-knobs 151 result = cursor.execute("select value from knobs where name=?", (knob_name,)).fetchone()
9983-knobs 152 if result:
9983-knobs 153 cursor.execute("update knobs set value=? where name=?", (knob_value, knob_name,))
9983-knobs 154 else:
9983-knobs 155 cursor.execute("insert into knobs(name, value) values(?, ?)", (knob_name, knob_value,))
9982-getdata 156
9982-getdata 157 self.conn.commit()
9982-getdata 158
9982-getdata 159 def get_latest_message_timestamp(self):
9982-getdata 160 cursor = self.cursor()
9982-getdata 161 result = cursor.execute("select timestamp from log order by timestamp desc limit 1").fetchone()
9982-getdata 162 if result:
9982-getdata 163 return result[0]
9982-getdata 164
genesis 165 def get_at(self, handle=None):
9985-single-thread 166 cursor = self.cursor()
genesis 167 at = []
genesis 168 if handle == None:
9979-presence 169 results = cursor.execute("select handle_id, address, port, updated_at, strftime('%s', updated_at) from at\
genesis 170 order by updated_at desc").fetchall()
genesis 171 else:
9985-single-thread 172 result = cursor.execute("select handle_id from handles where handle=?",
9992-handle-edge-... 173 (handle,)).fetchone()
9992-handle-edge-... 174 if None != result:
9992-handle-edge-... 175 handle_id = result[0]
genesis 176 else:
genesis 177 return []
9979-presence 178 results = cursor.execute("select handle_id, address, port, updated_at, strftime('%s', updated_at) from at \
9992-handle-edge-... 179 where handle_id=? order by updated_at desc",
9992-handle-edge-... 180 (handle_id,)).fetchall()
genesis 181 for result in results:
9979-presence 182 handle_id, address, port, updated_at_utc, updated_at_unixtime = result
9985-single-thread 183 h = cursor.execute("select handle from handles where handle_id=?",
genesis 184 (handle_id,)).fetchone()[0]
9979-presence 185 if updated_at_utc:
9979-presence 186 dt_format = '%Y-%m-%d %H:%M:%S.%f'
9979-presence 187 dt_utc = datetime.datetime.strptime(updated_at_utc, dt_format)
9979-presence 188 dt_local = self.utc_to_local(dt_utc)
9979-presence 189 updated_at = datetime.datetime.strftime(dt_local, dt_format)
9979-presence 190 else:
9979-presence 191 updated_at = "no packets received from this address"
9979-presence 192
9979-presence 193 at.append({
9979-presence 194 "handle": h,
9979-presence 195 "address": "%s:%s" % (address, port),
9979-presence 196 "active_at": updated_at,
9979-presence 197 "active_at_unixtime": int(updated_at_unixtime) if updated_at_unixtime else 0
9979-presence 198 })
genesis 199 return at
genesis 200
genesis 201 def import_at_and_wot(self, at_path):
9985-single-thread 202 cursor = self.cursor()
9985-single-thread 203 wot = imp.load_source('wot', at_path)
9985-single-thread 204 for peer in wot.peers:
9985-single-thread 205 results = cursor.execute("select * from handles where handle=? limit 1",
9985-single-thread 206 (peer["name"],)).fetchall()
9985-single-thread 207 if len(results) == 0:
9985-single-thread 208 key = peer["key"]
9985-single-thread 209 port = peer["port"]
9985-single-thread 210 address = peer["address"]
9985-single-thread 211 cursor.execute("insert into wot(peer_id) values(null)")
9985-single-thread 212 peer_id = cursor.lastrowid
9985-single-thread 213 cursor.execute("insert into handles(peer_id, handle) values(?, ?)",
9985-single-thread 214 (peer_id, peer["name"]))
9985-single-thread 215 handle_id = cursor.lastrowid
9985-single-thread 216 cursor.execute("insert into at(handle_id, address, port, updated_at) values(?, ?, ?, ?)",
9985-single-thread 217 (handle_id, peer["address"], peer["port"], None))
9985-single-thread 218 cursor.execute("insert into keys(peer_id, key) values(?, ?)",
9985-single-thread 219 (peer_id, key))
9985-single-thread 220 self.conn.commit()
genesis 221
9987-embargoing 222 def update_at(self, peer, set_active_at=True):
9985-single-thread 223 cursor = self.cursor()
9985-single-thread 224 row = cursor.execute("select handle_id from handles where handle=?",
9985-single-thread 225 (peer["handle"],)).fetchone()
9985-single-thread 226 if row != None:
9985-single-thread 227 handle_id = row[0]
9985-single-thread 228 else:
9984-unbork-at-co... 229 raise Exception("handle not found")
9985-single-thread 230
9984-unbork-at-co... 231 at_entry = cursor.execute("select handle_id, address, port from at where handle_id=?",
9984-unbork-at-co... 232 (handle_id,)).fetchone()
9984-unbork-at-co... 233
9984-unbork-at-co... 234
9979-presence 235 timestamp = datetime.datetime.utcnow() if set_active_at else None
9984-unbork-at-co... 236 if at_entry == None:
9984-unbork-at-co... 237 cursor.execute("insert into at(handle_id, address, port, updated_at) values(?, ?, ?, ?)",
9984-unbork-at-co... 238 (handle_id,
9984-unbork-at-co... 239 peer["address"],
9984-unbork-at-co... 240 peer["port"],
9984-unbork-at-co... 241 timestamp))
9984-unbork-at-co... 242 logging.debug("inserted new at entry for %s: %s:%d" % (
9984-unbork-at-co... 243 peer['handle'],
9984-unbork-at-co... 244 peer['address'],
9984-unbork-at-co... 245 peer['port']))
9984-unbork-at-co... 246
9982-getdata 247
9984-unbork-at-co... 248 else:
9984-unbork-at-co... 249 try:
9982-getdata 250 cursor.execute("update at set updated_at = ?,\
9982-getdata 251 address = ?,\
9982-getdata 252 port = ?\
9982-getdata 253 where handle_id=?",
9982-getdata 254 (timestamp,
9982-getdata 255 peer["address"],
9982-getdata 256 peer["port"],
9982-getdata 257 handle_id))
9982-getdata 258
9984-unbork-at-co... 259 except sqlite3.IntegrityError:
9984-unbork-at-co... 260 cursor.execute("delete from at where handle_id=?", (handle_id,))
9984-unbork-at-co... 261
9984-unbork-at-co... 262
9985-single-thread 263 self.conn.commit()
genesis 264
genesis 265 def add_peer(self, handle):
9985-single-thread 266 cursor = self.cursor()
9985-single-thread 267 cursor.execute("insert into wot(peer_id) values(null)")
9984-unbork-at-co... 268 peer_id = cursor.lastrowid
9985-single-thread 269 cursor.execute("insert into handles(peer_id, handle) values(?, ?)",
9985-single-thread 270 (peer_id, handle))
9985-single-thread 271 self.conn.commit()
9991-improved-log... 272
genesis 273
genesis 274 def remove_peer(self, handle):
9985-single-thread 275 cursor = self.cursor()
9985-single-thread 276
9985-single-thread 277
9985-single-thread 278 result = cursor.execute("select peer_id from handles where handle=?",
9985-single-thread 279 (handle,)).fetchone()
9985-single-thread 280 if result == None:
9984-unbork-at-co... 281 raise Exception("handle not found")
9985-single-thread 282 else:
9985-single-thread 283 peer_id = result[0]
9985-single-thread 284
genesis 285
9985-single-thread 286 handle_ids = self.get_handle_ids_for_peer(peer_id)
9985-single-thread 287 for handle_id in handle_ids:
9985-single-thread 288
genesis 289
9985-single-thread 290 cursor.execute("delete from at where handle_id=?", (handle_id,))
genesis 291
9985-single-thread 292 cursor.execute("delete from handles where peer_id=?", (peer_id,))
genesis 293
9985-single-thread 294
genesis 295
9985-single-thread 296 cursor.execute("delete from keys where peer_id=?", (handle_id,))
9985-single-thread 297
9985-single-thread 298
9985-single-thread 299
9985-single-thread 300 cursor.execute("delete from wot where peer_id=?", (peer_id,))
9985-single-thread 301 self.conn.commit()
genesis 302
genesis 303
genesis 304 def add_key(self, handle, key):
9985-single-thread 305 cursor = self.cursor()
9985-single-thread 306 peer_id = cursor.execute("select peer_id from handles where handle=?", (handle,)).fetchone()[0]
9985-single-thread 307 if peer_id != None:
9985-single-thread 308 cursor.execute("insert into keys(peer_id, key) values(?, ?)", (peer_id, key))
9985-single-thread 309 self.conn.commit()
genesis 310
genesis 311 def remove_key(self, key):
9985-single-thread 312 cursor = self.cursor()
9985-single-thread 313 cursor.execute("delete from keys where key=?", (key,))
9985-single-thread 314 self.conn.commit()
genesis 315
genesis 316 def get_handle_ids_for_peer(self, peer_id):
9985-single-thread 317 cursor = self.cursor()
9985-single-thread 318 return list(chain.from_iterable(cursor.execute("select handle_id from handles where peer_id=?",
genesis 319 (peer_id,)).fetchall()))
genesis 320
9989-show-wot-nicks 321 def get_peer_handles(self):
9985-single-thread 322 cursor = self.cursor()
9985-single-thread 323 handles = self.listify(cursor.execute("select handle from handles").fetchall())
9989-show-wot-nicks 324 return handles
9989-show-wot-nicks 325
genesis 326 def get_peers(self):
9985-single-thread 327 cursor = self.cursor()
genesis 328 peers = []
9985-single-thread 329 handles = cursor.execute("select handle from handles").fetchall()
genesis 330
genesis 331 for handle in handles:
genesis 332 peer = self.get_peer_by_handle(handle[0])
9992-handle-edge-... 333 if not (self.is_duplicate(peers, peer)):
genesis 334 peers.append(peer)
genesis 335 return peers
genesis 336
9987-embargoing 337 def listify(self, results):
9987-embargoing 338 return list(chain.from_iterable(results))
9982-getdata 339
9982-getdata 340 def log_has_message(self, message_hash):
9982-getdata 341 cursor = self.cursor()
9982-getdata 342 result = cursor.execute("select exists(select 1 from log where message_hash=?)\
9982-getdata 343 limit 1", (binascii.hexlify(message_hash),)).fetchone()
9982-getdata 344 return result[0]
9982-getdata 345
9982-getdata 346 def log_message(self, message):
9982-getdata 347 cursor = self.cursor()
9982-getdata 348 message_hash_hex_string = binascii.hexlify(message.message_hash)
9982-getdata 349 cursor.execute("insert into log(message_hash, message_bytes, command, timestamp) values(?, ?, ?, ?)",
9982-getdata 350 (message_hash_hex_string,
9982-getdata 351 buffer(message.message_bytes),
9982-getdata 352 message.command,
9982-getdata 353 message.timestamp))
9982-getdata 354 self.conn.commit()
9982-getdata 355
9982-getdata 356 def get_message(self, message_hash):
9982-getdata 357 cursor = self.cursor()
9982-getdata 358 message_hash_hex_string = binascii.hexlify(message_hash)
9982-getdata 359 result = cursor.execute("select command, message_bytes from log where message_hash=? limit 1",
9982-getdata 360 (message_hash_hex_string,)).fetchone()
9982-getdata 361 if result:
9982-getdata 362 return result[0], result[1][:]
9982-getdata 363
9982-getdata 364 return None, None
9982-getdata 365
9982-getdata 366 def get_keyed_peers(self, exclude_addressless=False, exclude_ids=[]):
9985-single-thread 367 cursor = self.cursor()
9982-getdata 368 peer_ids = self.listify(cursor.execute("select peer_id from keys\
9982-getdata 369 where peer_id not in (%s) order by random()" % ','.join('?'*len(exclude_ids)),
9982-getdata 370 exclude_ids).fetchall())
9987-embargoing 371 peers = []
9987-embargoing 372 for peer_id in peer_ids:
9985-single-thread 373 handle = cursor.execute("select handle from handles where peer_id=?", (peer_id,)).fetchone()[0]
9987-embargoing 374 peer = self.get_peer_by_handle(handle)
9986-rebroadcast-... 375 if self.is_duplicate(peers, peer):
9986-rebroadcast-... 376 continue
9984-unbork-at-co... 377 if exclude_addressless and (peer.address == None or peer.port == None):
9986-rebroadcast-... 378 continue
9986-rebroadcast-... 379 peers.append(peer)
9987-embargoing 380 return peers
9987-embargoing 381
9979-presence 382
9979-presence 383 def handle_is_online(self, handle):
9979-presence 384
9979-presence 385
9979-presence 386 at = self.get_at(handle)[0]
9979-presence 387 if at["active_at_unixtime"] > time.time() - int(self.get_knob("peer_offline_interval_seconds")):
9979-presence 388 return True
9979-presence 389 else:
9979-presence 390 return False
9979-presence 391
9979-presence 392 def utc_to_local(self, utc_dt):
9979-presence 393
9979-presence 394 timestamp = calendar.timegm(utc_dt.timetuple())
9979-presence 395 local_dt = datetime.datetime.fromtimestamp(timestamp)
9979-presence 396 assert utc_dt.resolution >= datetime.timedelta(microseconds=1)
9979-presence 397 return local_dt.replace(microsecond=utc_dt.microsecond)
9979-presence 398
9979-presence 399 def handle_is_away(self, handle):
9979-presence 400
9979-presence 401 cursor = self.cursor()
9979-presence 402 away_interval_seconds = int(self.get_knob("peer_away_interval_seconds"))
9979-presence 403 dt = datetime.datetime.utcfromtimestamp(
9979-presence 404 time.time() - away_interval_seconds
9979-presence 405 )
9979-presence 406 raw_messages = cursor.execute("select message_bytes from log where created_at > ?", (dt,)).fetchall()
9979-presence 407 for message_bytes in raw_messages:
9979-presence 408 int_ts, self_chain, net_chain, speaker, body = Message._unpack_message(message_bytes[0][:])
9979-presence 409 if speaker == handle:
9979-presence 410 return False
9979-presence 411 return True
9979-presence 412
genesis 413 def get_peer_by_handle(self, handle):
9985-single-thread 414 cursor = self.cursor()
9985-single-thread 415 handle_info = cursor.execute("select handle_id, peer_id from handles where handle=?",
genesis 416 (handle,)).fetchone()
genesis 417
genesis 418 if handle_info == None:
genesis 419 return None
genesis 420
9982-getdata 421 peer_id = handle_info[1]
9985-single-thread 422 address = cursor.execute("select address, port from at where handle_id=?\
genesis 423 order by updated_at desc limit 1",
genesis 424 (handle_info[0],)).fetchone()
9985-single-thread 425 handles = self.listify(cursor.execute("select handle from handles where peer_id=?",
9982-getdata 426 (peer_id,)).fetchall())
9985-single-thread 427 keys = self.listify(cursor.execute("select key from keys where peer_id=?\
9982-getdata 428 order by random()",
9982-getdata 429 (peer_id,)).fetchall())
9982-getdata 430 return Peer(self.station.socket, {
genesis 431 "handles": handles,
genesis 432 "peer_id": handle_info[1],
9986-rebroadcast-... 433 "address": address[0] if address else None,
9986-rebroadcast-... 434 "port": address[1] if address else None,
genesis 435 "keys": keys
genesis 436 })
9987-embargoing 437
genesis 438 def is_duplicate(self, peers, peer):
genesis 439 for existing_peer in peers:
9982-getdata 440 if (not existing_peer.address is None
9982-getdata 441 and existing_peer.address == peer.address
9982-getdata 442 and existing_peer.port == peer.port):
genesis 443 return True
genesis 444 return False