3.1.8 Telegram Group Monitoring and Reporting Bug Fixing Release

lexicon_negative_english.txt

@@ -1,4 +1,3 @@
-2-dimensional
 2-faced
 2-faces
 abase
@@ -4718,7 +4717,6 @@ narrower
 nastily
 nastiness
 nasty
-natural
 naturism
 naughty
 nauseate

lexicon_positive_russian.txt

@@ -1250,7 +1250,6 @@
 миролюбивый
 миролюбие
 миротворческий
-младший
 многогранность
 многогранный
 многозначительность

php/agent/agent_chat.php

@@ -95,7 +95,7 @@ function test_findchat() {
 	say("there is сотрудник, человек мария ивановна сидорова, комната 123, отдел кадров.");
 	say("there is покупка, товар кроссовки, назначение для туризма, магазин рыбалка и охота.");
 	say("there is покупка, товар кроссовки, назначение для бега, магазин спортмастер.");
-	say("there is покупка,  товар кроссовки, назначение для бега, магазин чемпион.");
+	say("there is покупка, товар кроссовки, назначение для бега, магазин чемпион.");
 	say("there is новость, текст трамп объявил санкции, дата сегодня, источник lenta.");
 	say("there is новость, текст трамп обвинил макрона, дата сегодня, источник kommersant.");
 	say("there is новость, текст трамп обвинил меркель, дата вчера, источник lenta.");
@@ -153,6 +153,28 @@ function test_findchat() {
 	get("Дата вчера, источник kommersant, текст трамп поздравил путина?");
 	say("да");
 	get("Ok.");
+
+    // making sure that sentiment is ignored when searching things
+    say("новость has sentiment");
+	say("текст трамп объявил санкции sentiment 1.");
+	say("текст трамп обвинил макрона sentiment 1.");
+	say("текст трамп обвинил меркель sentiment 1.");
+	say("текст трамп поздравил путина sentiment 1.");
+	say("текст трамп обвинил байдена sentiment 2.");
+	say("текст трамп выступил в конгрессе sentiment 2.");
+	say("текст трамп отправился в поездку sentiment 2.");
+	say("what текст трамп отправился в поездку текст, sentiment?");
+	get("There sentiment 2, текст трамп отправился в поездку.");
+	say("what sentiment 1?");
+	get("There is новость, sentiment 1, дата вчера, источник kommersant, текст трамп поздравил путина; is новость, sentiment 1, дата вчера, источник lenta, текст трамп обвинил меркель; is новость, sentiment 1, дата сегодня, источник kommersant, текст трамп обвинил макрона; is новость, sentiment 1, дата сегодня, источник lenta, текст трамп объявил санкции.");
+	say("меня интересует президент трамп");
+	get("Источник kommersant или lenta?");
+	say("lenta");
+	get("Дата 2020-08-07 или вчера или сегодня?");
+	say("вчера");
+	get("Дата вчера, источник lenta, текст трамп обвинил меркель?");
+	say("Да");
+	get("Ok.");
 
 	//variations
 	say("нужно что-нибудь для туризма или для бега");

php/agent/test_api.php

@@ -25,7 +25,7 @@
 
 include_once("pest.php");
 
-$version = "3.1.6";
+$version = "3.1.8";
 $copyright = " Copyright © 2020 Anton Kolonin, Aigents®.";
 
 $baseURL = "http://localhost:1180/?";

src/main/java/net/webstructor/agent/Body.java

@@ -65,7 +65,7 @@
 public abstract class Body extends Anything implements Environment, Updater
 {
 	public final static String APPNAME = "Aigents";
-	public final static String VERSION = "3.1.6";
+	public final static String VERSION = "3.1.8";
 	public final static String COPYRIGHT = "Copyright © 2020 Anton Kolonin, Aigents®.";
 	public final static String ORIGINSITE = "https://aigents.com";
 	public final static String DEFAULT_API_URL = "/al";

src/main/java/net/webstructor/comm/Mediator.java

@@ -106,15 +106,25 @@ protected void mergePeer(Thing merger, Thing mergee, String[] names) {
 			body.error(Writer.capitalize(name)+" fails merging "+mergee,e);
 		}
 	}
+
+	protected String groupFullName(String group_name) {
+		return name + ":" + group_name;
+	}
+
+	protected String messageLink(String group_usename, String group_title, String group_id, String message_id) {
+//TODO: fix hack and find solution for Slack!
+		return groupFullName(group_title);
+	}
 
 //TODO: move to Grouper under Conversation scope for Slack and WeChat unification 
 	// - adding session attributes?
 	// - adding dedicated unauthorized chat sessions?
-	protected void updateGroup(String group_id, String group_name, String peer_id, boolean is_in, boolean is_bot, String text){
+	protected void updateGroup(String message_id, String group_id, String group_username, String group_title, String peer_id, boolean is_in, boolean is_bot, String text){
 		try {
 			//1) get group by id (eg. "telegram_id")
 			String name_id = name+" id";
-			String full_group_name = name + ":" + group_name;
+			String full_group_name = groupFullName(group_title);
+			String message_link = messageLink(group_username, group_title, group_id, message_id);
 			Collection g = body.storager.getByName(name_id, group_id);
 			Thing group;
 			if (!AL.empty(g)){
@@ -126,7 +136,7 @@ protected void updateGroup(String group_id, String group_name, String peer_id, b
 				group.setString(name_id,group_id);
 				group.storeNew(body.storager);
 			}
-body.debug(Writer.capitalize(name)+" channel name_id "+name_id+" group_name "+group_name+" group "+group.toString()+" peer_id "+peer_id+" text "+text);//TODO: remove debug
+body.debug(Writer.capitalize(name)+" channel name_id "+name_id+" name"+group_username+" title "+group_title+" group "+group.toString()+" peer_id "+peer_id+" text "+text);//TODO: remove debug
 			//3) if (!is_bot), add/remove peer to/from group
 			if (!is_bot){//TODO: handle bots as well!?
 				//4) get peer by id (eg. "telegram_id")
@@ -163,7 +173,7 @@ protected void updateGroup(String group_id, String group_name, String peer_id, b
 				Date today = Time.today(0);
 				Iter parse = new Iter(Parser.parse(text));
 				if (!AL.empty(topics)) for (Iterator tit = topics.iterator(); tit.hasNext();)
-					matcher.match(parse, null, (Thing)tit.next(), today, full_group_name, null, thingPaths, null, null, null);
+					matcher.match(parse, null, (Thing)tit.next(), today, message_link, null, thingPaths, null, null, null);
 				//8) send update if topic is matched
 //TODO: exclude sender in the news update
 				body.getPublisher().update(null,today,thingPaths,true,group);//forced

src/main/java/net/webstructor/comm/Slacker.java

@@ -215,7 +215,8 @@ public boolean handleHTTP(HTTPeer parent, String url, String header, String requ
 					String channel_name = groupName(channel);
 					//TODO:2) check if user is bot
 					boolean is_bot = false;//TODO check if is bot
-					updateGroup(channel, channel_name, user, true, is_bot, text);
+//TODO: message_id
+					updateGroup(null, channel, channel_name, channel_name, user, true, is_bot, text);
 
 //TODO: be able to do unauthorized group conversations and enable chat message handling!
 

src/main/java/net/webstructor/comm/Socializer.java

@@ -382,6 +382,29 @@ public final String cacheReport(SocialFeeder feeder, HashMap feeds, String user_
 		my_posts_for_the_period = "my posts for the period",
 		reputation = "reputation in community",
 		social_graph = "social graph";
+/*
+? ``all connections`` - all other users connected my means of communications to the current user **(cross peers != null)**
+- ``my interests`` - **default**, clusters of the posts/messages corresponding to interests of the current user, labeled by keywords typical to respective posts/messages
+- ``interests of my friends`` - **default**, clusters of the posts/messages corresponding to interests of other users excluding the  current user, labeled by keywords typical to respective posts/messages
+- ``similar to me`` - **default**, other users ranked by simlarity in respect to the current user
+- ``best friends`` - **default**, other users that are the most involved in mutual communications (likes, votes, comments and mentions) with the current user
+- ``fans`` - **default**, other users that are the most involved in communications (likes, votes, comments and mentions) directed to the current user but not the other way around
+- ``like and comment me`` - other users who provide likes and votes in respect to the current user
+- ``authorities`` - **default**, other users that are getting the most of communications (likes, votes, comments and mentions) from the current user
+- ``reputation`` - **default**, list of the users with highest reputation score within entire reachable community, including any users visible given privacy restrictions
+- ``social graph`` - **default**, rendering of the nearest social environment in form of interactive social graph 
+- ``liked by me`` - other users that are getting the most of votes and likes from the current user
+- ``my karma by periods`` - dynamic of the "karma" (social capital) for the current user
+- ``my words by periods`` - **default**, words used by the current user getting most of attention (likes, votes, comments and mentions) from other users, broken down by time periods
+- ``my friends by periods`` - **default**, other users paying attention (likes, votes, comments and mentions) to the current user, broken down by time periods
+- ``my favorite words`` - **default**, words most oftenly used, liked and commented by the current user
+- ``my posts liked and commented`` - **default**, posts of the current user most oftenly liked and commented by the other users
+- ``my best words`` - words most oftenly liked and commented by all of the users including the current user 
+- ``my words liked and commented`` - words of the current user most oftenly liked and commented by the other users
+- ``words liked by me`` - words most oftenly liked by the current user 
+- ``words of my friends`` - **default**, words most oftenly used by users other than current user
+- ``my posts for the period`` - **default**, all posts by the current user for given period
+*/
 	static final String[] report_options = {
 		my_interests,
 		interests_of_my_friends,

src/main/java/net/webstructor/comm/telegram/Telegram.java

@@ -192,14 +192,16 @@ public void save() {
 		}
 	}
 
-	protected void updateInteraction(Date date, String group_id, String group_name, String message_id, String reply_to_message_id, String from_id, String reply_to_from_id, java.util.Set<String> mention_ids, String text_body) {
+	protected void updateInteraction(Date date, String group_id, String group_name, String message_id, String reply_to_message_id, String from_id, String reply_to_from_id, java.util.Set<String> mention_ids, String text) {
 		if (debug)
 //TODO: debug
 			body.debug("Telegram crawling from "+from_id+" to "+reply_to_from_id+" mentions "+mention_ids);
 
+		text = text.replace('\n', ' ');
+
 		String to_id = !AL.empty(reply_to_from_id) ? reply_to_from_id : group_id;
-		int intvalue = text_body != null ? text_body.length() : 1;//empty comment still counts
-		String weight = text_body != null ? String.valueOf(intvalue) : null;
+		int intvalue = text != null ? text.length() : 1;//empty comment still counts
+		String weight = text != null ? String.valueOf(intvalue) : null;
 		//https://t.me/agirussia/8855 (public - agirussia, 8855)
 		//https://t.me/c/1410910487/75 (private -1001410910487, 75)
 		//String url = !AL.empty(group_name) ? "https://t.me/" + group_name + "/" + message_id : null; 
@@ -208,9 +210,9 @@ protected void updateInteraction(Date date, String group_id, String group_name,
 
 		//comments
 		//mentions
-		int logvalue = 1 + (AL.empty(text_body) ? 0 : (int)Math.round(Math.log10(text_body.length())));
+		int logvalue = 1 + (AL.empty(text) ? 0 : (int)Math.round(Math.log10(text.length())));
 		if (logger != null)
-			SocialCacher.write(logger, name, date, date.getTime(), "comment", from_id, to_id, weight, null, permlink, parent_permlink, group_name/*title*/, text_body, null, null);
+			SocialCacher.write(logger, name, date, date.getTime(), "comment", from_id, to_id, weight, null, permlink, parent_permlink, group_name/*title*/, text, null, null);
 		if (!AL.empty(reply_to_from_id)) {
 			updateInteraction(date,"comments",from_id,reply_to_from_id,logvalue);//update from->reply_to_from
 		}

src/main/java/net/webstructor/comm/telegram/Telegrammer.java

@@ -253,6 +253,22 @@ private void delete(String chat_id, String msg_id) throws IOException {
 			body.error("Telegram error",e);
 		}
 	}
+
+	@Override
+	protected String messageLink(String group_usename, String group_title, String group_id, String message_id) {
+		//https://stackoverflow.com/questions/51065460/link-message-by-message-id-via-telegram-bot
+		if (group_usename != null) {
+			//https://t.me/agirussia/8855 (public - agirussia, 8855)
+			return "https://t.me/"+group_usename+"/"+message_id;
+		}
+		if (group_id.startsWith("-100")) {
+			//https://t.me/c/1410910487/75 (private -1001410910487, 75)
+			group_id = group_id.substring(3);
+			return "https://t.me/c/"+group_id+"/"+message_id;
+		}
+		//if not group "type":"supergroup" !?
+		return super.messageLink(group_usename, group_title, group_id, message_id);
+	}
 
 	private long handle(String response) throws IOException{
 		String botname = body.getSelf().getString(Body.telegram_name);
@@ -355,7 +371,7 @@ private long handle(String response) throws IOException{
 
 				if (!from_id.equals(chat_id)){
 					boolean is_bot = from.containsKey("is_bot")? from.getBoolean("is_bot") : false;
-					updateGroup(chat_id, chat_title, from_id, true, is_bot, text);
+					updateGroup(message_id, chat_id, chat_username, chat_title, from_id, true, is_bot, text);
 
 					//process group interactions
 					if (telegram != null) {

src/main/java/net/webstructor/comm/twitter/Twitter.java

@@ -70,7 +70,7 @@ public TwitterFeeder(Environment body, Twitter api, String user_id, LangPack lan
 		this.api = api;
 	}
 	public void getFeed(String id, String token, String token_secret, Date since, Date until, StringBuilder detail) throws IOException {
-		body.debug("Twitter crawling "+user_id);
+		body.debug("Twitter crawling "+id);
 
 		//https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines
 		//https://developer.twitter.com/en/docs/tweets/timelines/api-reference/get-statuses-home_timeline
@@ -81,7 +81,20 @@ public void getFeed(String id, String token, String token_secret, Date since, Da
 		try {
 //TODO deal with rate limits, if needed
 //TODO if date limit is not hit, iterate with max_id-1
-			String[][] params = new String[][] {
+
+			String[][] params = !Str.isLong(id) ?
+				new String[][] {
+					//new String [] {"user_id","563516882"}, 
+					new String [] {"screen_name",id}, 
+					new String [] {"count",String.valueOf(count)},
+					//new String [] {"since_id","false"},
+					//new String [] {"max_id","1257053173916659712"},
+					new String [] {"tweet_mode", "extended"},
+					new String [] {"trim_user","false"},
+					new String [] {"exclude_replies","false"},
+					new String [] {"include_rts","true"}
+				}:
+				new String[][] {
 				//new String [] {"user_id","563516882"}, 
 				//new String [] {"screen_name","bengoertzel"}, 
 				new String [] {"count",String.valueOf(count)},

src/main/java/net/webstructor/core/Property.java

@@ -23,6 +23,7 @@
  */
 package net.webstructor.core;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -53,7 +54,8 @@ public class Property extends Anything implements Named {
 	protected String name;//TODO: is
 	protected Anything owner;
 	protected Storager storager = null;
-	private boolean hasPatterns = false;
+	private ArrayList<Set> patterns = null;
+
 	private int limit = DEFAULT_PROPERTY_LIMIT;
 
 	public Property(Anything owner,String name,String def) {
@@ -98,38 +100,46 @@ public String name() {
 	private static final Pattern number = Pattern.compile("[-+−]?[0-9]+([\\.,][0-9]+)?");
 
 	public boolean hasPatterns() {
-		return hasPatterns;
+		return patterns != null;
 	}
 
-	//TODO: build pre-compiled ordered list of patterns and keep in variable!?
 	private void compile() {
 		if (storager == null)
 			return;
 		Thing variable = storager.getThing(name);
-		if (variable == null)
-			return;
-		if (!AL.empty(variable.getThings(AL.patterns))){
-			hasPatterns = true;
-			return;
-		}
-		Collection is = variable.getThings(AL.is);
-		if (!AL.empty(is))
-			for (Iterator i = is.iterator(); i.hasNext();)
-				if (!AL.empty(((Thing)i.next()).getThings(AL.patterns))){
-					hasPatterns = true;
-					return;
+		compile(variable);
+	}
+
+	private boolean compile(Thing domain) {
+		boolean compiled = false;
+		if (domain != null) {
+			Collection ps = domain.getThings(AL.patterns);
+			if (!AL.empty(ps)) for (Object p : ps) {
+				Thing t = (Thing)p;
+				String patstr = t.name();
+				if (!AL.empty(patstr)) {
+					if (patterns == null)
+						patterns = new ArrayList<Set>(1);
+					Set pat = Reader.pat(storager, owner instanceof Thing ? (Thing)owner : null, patstr);
+					patterns.add(pat);
+					compiled = true;
 				}
+			}
+			Collection is = domain.getThings(AL.is);
+			if (!AL.empty(is))
+				for (Iterator i = is.iterator(); i.hasNext();)
+					if (compile((Thing)i.next()))
+						compiled = true;
+		}
+		return compiled;
 	}
 
 	public boolean read(Iter it, StringBuilder summary){
 		if (storager == null)
 			return false;
 		StringBuilder value = new StringBuilder();
 		//if variable has patterns, recursively to variable domains
-		Thing variable = storager.getThing(name);
-		if (variable == null)
-			return false;
-		if (read(it, variable, value)){
+		if (readVariable(it, value)){
 			//TODO: entity extraction goes here!?
 			String v = value.toString();
 			setString(v);
@@ -139,27 +149,16 @@ public boolean read(Iter it, StringBuilder summary){
 		return false;
 	}
 
-	//TODO: move to static Reader, once patterns are pre-compiled
-	public boolean read(Iter it, Thing domain, StringBuilder summary){
-		//first, check patterns of the domain
-		Collection ps = domain.getThings(AL.patterns);
-		if (!AL.empty(ps)){
-			for (Iterator i = ps.iterator(); i.hasNext();){
-				//TODO: system-wide cache of compiled patterns!?
-				String patstr = ((Thing)i.next()).name();
-				Set pat = Reader.pat(storager, owner instanceof Thing ? (Thing)owner : null, patstr);
-				if (Reader.read(it, pat, summary))
-					return true;
-			}
-		}
-		//next, check if domain has super-domains with patterns
-		Collection domains = (Collection)domain.get(AL.is);
-		if (!AL.empty(domains)){
-			for (Iterator i = domains.iterator(); i.hasNext();){
-				if (read(it, (Thing)i.next(), summary))
+	private boolean readVariable(Iter it, StringBuilder summary){
+		Thing variable = storager.getThing(name);
+		if (variable == null)
+			return false;
+		if (!AL.empty(patterns))//precompiled patterns
+			for (Set pat : patterns) {
+				boolean read = Reader.read(it, pat, summary);
+				if (read)
 					return true;
 			}
-		}
 		return false;
 	}
 

src/main/java/net/webstructor/data/Emotioner.java

@@ -32,9 +32,9 @@ class Emoticon {
 //https://lemire.me/blog/2018/06/15/emojis-java-and-strings/
 //https://www.branah.com/unicode-converter
 public class Emotioner {
-	public static final String positive = "\ud83d\ude0a";//😊0001f60a
-	public static final String negative = "\ud83d\ude1e";//😞0001f61e
-	public static final String flushed = "\ud83d\ude33";//😳0001F633 
+	public static final String positive = "\ud83d\ude0a";//😊0001f60a//smiling face with smiling eyes
+	public static final String negative = "\ud83d\ude1e";//😞0001f61e//disappointed face
+	public static final String flushed = "\ud83d\ude33";//😳0001F633//flushed face
 	public static final String emotion(int s) {//sentiment
 		return s < -50 ? Emotioner.negative : s > 50 ? Emotioner.positive : "";
 	}