edavis.dev
this repo has no description
feeds/battle.py: treat length and lang as a pair
+42
-34
+37
-34
feeds/battle.py
+37
-34
feeds/battle.py
···
18
18
self.db_cnx.execute("""
19
19
create table if not exists posts (
20
20
uri text,
21
-
grapheme_length integer unique,
21
+
grapheme_length integer,
22
22
create_ts timestamp,
23
23
lang text
24
24
);
25
+
create unique index if not exists ll_idx on posts(grapheme_length, lang);
25
26
""")
26
27
27
28
self.logger = logging.getLogger('feeds.battle')
···
42
43
repo = commit['repo']
43
44
path = op['path']
44
45
post_uri = f'at://{repo}/{path}'
45
-
l = grapheme.length(record['text'])
46
+
length = grapheme.length(record['text'])
46
47
ts = self.safe_timestamp(record['createdAt']).timestamp()
47
48
48
49
self.transaction_begin(self.db_cnx)
···
52
53
self.db_cnx.execute("""
53
54
insert into posts(uri, grapheme_length, create_ts, lang)
54
55
values(:uri, :length, :ts, :lang)
55
-
on conflict(grapheme_length) do update set uri = :uri
56
-
""", dict(uri=post_uri, length=l, ts=ts, lang=lang))
56
+
on conflict do update set uri = :uri, create_ts = :ts
57
+
""", dict(uri=post_uri, length=length, ts=ts, lang=lang))
57
58
58
59
def commit_changes(self):
59
60
self.logger.debug('committing changes')
···
61
62
self.wal_checkpoint(self.db_cnx, 'RESTART')
62
63
63
64
def serve_feed(self, limit, offset, langs):
64
-
# if '*' in langs:
65
-
# cur = self.db_cnx.execute(
66
-
# "select uri from posts order by grapheme_length asc limit :limit offset :offset",
67
-
# dict(limit=limit, offset=offset)
68
-
# )
69
-
# return [uri for (uri,) in cur]
70
-
# else:
71
-
# lang_values = list(langs.values())
72
-
# lang_selects = ['select uri, grapheme_length from posts where lang = ?'] * len(lang_values)
73
-
# lang_clause = ' union '.join(lang_selects)
74
-
# cur = self.db_cnx.execute(
75
-
# lang_clause + ' order by grapheme_length asc limit ? offset ?',
76
-
# [*lang_values, limit, offset]
77
-
# )
78
-
# return [uri for (uri, create_ts) in cur]
65
+
if '*' in langs:
66
+
cur = self.db_cnx.execute("""
67
+
select uri
68
+
from posts
69
+
order by grapheme_length asc
70
+
limit :limit offset :offset
71
+
""", dict(limit=limit, offset=offset))
72
+
return [uri for (uri,) in cur]
73
+
else:
74
+
lang_values = list(langs.values())
75
+
lang_selects = ['select uri, grapheme_length from posts where lang = ?'] * len(lang_values)
76
+
lang_clause = ' union '.join(lang_selects)
77
+
cur = self.db_cnx.execute(
78
+
lang_clause + ' order by grapheme_length asc limit ? offset ?',
79
+
[*lang_values, limit, offset]
80
+
)
81
+
return [uri for (uri, grapheme_length) in cur]
79
82
80
-
cur = self.db_cnx.execute("""
81
-
select uri
82
-
from posts
83
-
order by grapheme_length asc
84
-
limit :limit offset :offset
85
-
""", dict(limit=limit, offset=offset))
86
-
return [uri for (uri,) in cur]
83
+
def serve_feed_debug(self, limit, offset, langs):
84
+
if '*' in langs:
85
+
query = """
86
+
select *, unixepoch('now') - create_ts as age_seconds
87
+
from posts
88
+
order by grapheme_length asc
89
+
limit :limit offset :offset
90
+
"""
91
+
bindings = [limit, offset]
92
+
else:
93
+
lang_values = list(langs.values())
94
+
lang_selects = ["select *, unixepoch('now') - create_ts as age_seconds from posts where lang = ?"] * len(lang_values)
95
+
lang_clause = ' union '.join(lang_selects)
96
+
query = lang_clause + ' order by grapheme_length asc limit ? offset ?'
97
+
bindings = [*lang_values, limit, offset]
87
98
88
-
def serve_feed_debug(self, limit, offset, langs):
89
-
query = """
90
-
select *
91
-
from posts
92
-
order by grapheme_length asc
93
-
limit :limit offset :offset
94
-
"""
95
-
bindings = dict(limit=limit, offset=offset)
96
99
return apsw.ext.format_query_table(
97
100
self.db_cnx, query, bindings,
98
101
string_sanitize=2, text_width=9999, use_unicode=True
+5
feedweb.py
+5
feedweb.py
···
1
1
#!/usr/bin/env python3
2
2
3
3
from flask import Flask, request, jsonify
4
+
from werkzeug.datastructures import LanguageAccept
4
5
5
6
from feed_manager import feed_manager
6
7
from feeds.rapidfire import RapidFireFeed
···
28
29
langs = request.accept_languages
29
30
30
31
if request.args.get('debug', '0') == '1':
32
+
if request.args.getlist('langs'):
33
+
req_langs = request.args.getlist('langs')
34
+
langs = LanguageAccept([(l, 1) for l in req_langs])
35
+
31
36
headers = {'Content-Type': 'text/plain; charset=utf-8'}
32
37
debug = feed_manager.serve_feed_debug(feed_uri, limit, offset, langs)
33
38
return debug, headers