From c0cf42a41aad8ab137055b3851ad7f5a2daff16d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lukas=20F=C3=BClling?= <lukas@k40s.net>
Date: Mon, 25 Feb 2019 19:30:40 +0100
Subject: [PATCH] add config option to filter for language

---
 config.json  |  3 ++-
 functions.py |  6 +++---
 main.py      | 34 +++++++++++++++++++++-------------
 3 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/config.json b/config.json
index 4d459f2..e62fce2 100644
--- a/config.json
+++ b/config.json
@@ -1,5 +1,6 @@
 {
+    "lang": "en",
 	"site": "https://botsin.space",
 	"cw": null,
 	"learn_from_cw": false
-}
\ No newline at end of file
+}
diff --git a/functions.py b/functions.py
index 7e0d917..4363341 100755
--- a/functions.py
+++ b/functions.py
@@ -16,11 +16,11 @@ def make_sentence(output):
 
 	shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one
 	db = sqlite3.connect("toots-copy.db")
-	db.text_factory=str
+	db.text_factory = str
 	c = db.cursor()
 	if cfg['learn_from_cw']:
 		toots = c.execute("SELECT content FROM `toots` ORDER BY RANDOM() LIMIT 10000").fetchall()
-	else: 
+	else:
 		toots = c.execute("SELECT content FROM `toots` WHERE cw = 0 ORDER BY RANDOM() LIMIT 10000").fetchall()
 	toots_str = ""
 	for toot in toots:
@@ -88,4 +88,4 @@ def extract_toot(toot):
 	text = re.sub("https://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mastodon-style mentions back in
 	text = re.sub("https://([^/]+)/users/([^ ]+)", r"@\2@\1", text) #put pleroma-style mentions back in
 	text = text.rstrip("\n") #remove trailing newline
-	return text
\ No newline at end of file
+	return text
diff --git a/main.py b/main.py
index bbc586d..0d64962 100755
--- a/main.py
+++ b/main.py
@@ -43,7 +43,7 @@ if os.path.exists("clientcred.secret"):
 	cfg['secret'] = open("usercred.secret").read().rstrip("\n")
 	os.remove("clientcred.secret")
 	os.remove("usercred.secret")
-		
+
 
 if "client" not in cfg:
 	print("No application info -- registering application with {}".format(cfg['site']))
@@ -75,8 +75,8 @@ def extract_toot(toot):
 
 client = Mastodon(
 	client_id=cfg['client']['id'],
-	client_secret = cfg['client']['secret'], 
-	access_token=cfg['secret'], 
+	client_secret = cfg['client']['secret'],
+	access_token=cfg['secret'],
 	api_base_url=cfg['site'])
 
 me = client.account_verify_credentials()
@@ -106,6 +106,18 @@ patterns = {
 	"pid": re.compile(r"[^\/]+$"),
 }
 
+
+def insert_toot(oii, acc, post, cursor):  # extracted to prevent duplication
+	pid = patterns["pid"].search(oii['object']['id']).group(0)
+	cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
+		pid,
+		1 if (oii['object']['summary'] != None and oii['object']['summary'] != "") else 0,
+		acc.id,
+		oii['object']['id'],
+		post
+	))
+
+
 for f in following:
 	last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
 	if last_toot != None:
@@ -161,7 +173,7 @@ for f in following:
 			for oi in j['orderedItems']:
 				if oi['type'] != "Create":
 					continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
-				
+
 				# its a toost baby
 				content = oi['object']['content']
 				toot = extract_toot(content)
@@ -172,15 +184,11 @@ for f in following:
 							#we've caught up to the notices we've already downloaded, so we can stop now
 							#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
 							done = True
-					pid = patterns["pid"].search(oi['object']['id']).group(0)
-					c.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
-						pid,
-						1 if (oi['object']['summary'] != None and oi['object']['summary'] != "") else 0,
-						f.id,
-						oi['object']['id'],
-						toot
-						)
-					)
+					if cfg['lang']:
+						if oi['object']['contentMap'][cfg['lang']]:  # filter for language
+							insert_toot(oi, f, toot, c)
+					else:
+						insert_toot(oi, f, toot, c)
 					pass
 				except:
 					pass #ignore any toots that don't successfully go into the DB