diff --git a/lexicon_negative_english.txt b/lexicon_negative_english.txt index 56a4509..d66de1d 100644 --- a/lexicon_negative_english.txt +++ b/lexicon_negative_english.txt @@ -1,4 +1,3 @@ -2-dimensional 2-faced 2-faces abase @@ -4718,7 +4717,6 @@ narrower nastily nastiness nasty -natural naturism naughty nauseate diff --git a/lexicon_positive_russian.txt b/lexicon_positive_russian.txt index bdc1464..1811c86 100644 --- a/lexicon_positive_russian.txt +++ b/lexicon_positive_russian.txt @@ -1250,7 +1250,6 @@ миролюбивый миролюбие миротворческий -младший многогранность многогранный многозначительность diff --git a/php/agent/agent_chat.php b/php/agent/agent_chat.php index f855ed4..4b4ab30 100644 --- a/php/agent/agent_chat.php +++ b/php/agent/agent_chat.php @@ -95,7 +95,7 @@ function test_findchat() { say("there is сотрудник, человек мария ивановна сидорова, комната 123, отдел кадров."); say("there is покупка, товар кроссовки, назначение для туризма, магазин рыбалка и охота."); say("there is покупка, товар кроссовки, назначение для бега, магазин спортмастер."); - say("there is покупка, товар кроссовки, назначение для бега, магазин чемпион."); + say("there is покупка, товар кроссовки, назначение для бега, магазин чемпион."); say("there is новость, текст трамп объявил санкции, дата сегодня, источник lenta."); say("there is новость, текст трамп обвинил макрона, дата сегодня, источник kommersant."); say("there is новость, текст трамп обвинил меркель, дата вчера, источник lenta."); @@ -153,6 +153,28 @@ function test_findchat() { get("Дата вчера, источник kommersant, текст трамп поздравил путина?"); say("да"); get("Ok."); + + // making sure that sentiment is ignored when searching things + say("новость has sentiment"); + say("текст трамп объявил санкции sentiment 1."); + say("текст трамп обвинил макрона sentiment 1."); + say("текст трамп обвинил меркель sentiment 1."); + say("текст трамп поздравил путина sentiment 1."); + say("текст трамп обвинил байдена sentiment 2."); + say("текст трамп выступил в конгрессе sentiment 2."); + say("текст трамп отправился в поездку sentiment 2."); + say("what текст трамп отправился в поездку текст, sentiment?"); + get("There sentiment 2, текст трамп отправился в поездку."); + say("what sentiment 1?"); + get("There is новость, sentiment 1, дата вчера, источник kommersant, текст трамп поздравил путина; is новость, sentiment 1, дата вчера, источник lenta, текст трамп обвинил меркель; is новость, sentiment 1, дата сегодня, источник kommersant, текст трамп обвинил макрона; is новость, sentiment 1, дата сегодня, источник lenta, текст трамп объявил санкции."); + say("меня интересует президент трамп"); + get("Источник kommersant или lenta?"); + say("lenta"); + get("Дата 2020-08-07 или вчера или сегодня?"); + say("вчера"); + get("Дата вчера, источник lenta, текст трамп обвинил меркель?"); + say("Да"); + get("Ok."); //variations say("нужно что-нибудь для туризма или для бега"); diff --git a/php/agent/test_api.php b/php/agent/test_api.php index bcdaa1a..ffa44c3 100644 --- a/php/agent/test_api.php +++ b/php/agent/test_api.php @@ -25,7 +25,7 @@ include_once("pest.php"); -$version = "3.1.6"; +$version = "3.1.8"; $copyright = " Copyright © 2020 Anton Kolonin, Aigents®."; $baseURL = "http://localhost:1180/?"; diff --git a/src/main/java/net/webstructor/agent/Body.java b/src/main/java/net/webstructor/agent/Body.java index 2b54e34..6fef670 100644 --- a/src/main/java/net/webstructor/agent/Body.java +++ b/src/main/java/net/webstructor/agent/Body.java @@ -65,7 +65,7 @@ public abstract class Body extends Anything implements Environment, Updater { public final static String APPNAME = "Aigents"; - public final static String VERSION = "3.1.6"; + public final static String VERSION = "3.1.8"; public final static String COPYRIGHT = "Copyright © 2020 Anton Kolonin, Aigents®."; public final static String ORIGINSITE = "https://aigents.com"; public final static String DEFAULT_API_URL = "/al"; diff --git a/src/main/java/net/webstructor/comm/Mediator.java b/src/main/java/net/webstructor/comm/Mediator.java index cb73013..75f3f03 100644 --- a/src/main/java/net/webstructor/comm/Mediator.java +++ b/src/main/java/net/webstructor/comm/Mediator.java @@ -106,15 +106,25 @@ protected void mergePeer(Thing merger, Thing mergee, String[] names) { body.error(Writer.capitalize(name)+" fails merging "+mergee,e); } } + + protected String groupFullName(String group_name) { + return name + ":" + group_name; + } + + protected String messageLink(String group_usename, String group_title, String group_id, String message_id) { +//TODO: fix hack and find solution for Slack! + return groupFullName(group_title); + } //TODO: move to Grouper under Conversation scope for Slack and WeChat unification // - adding session attributes? // - adding dedicated unauthorized chat sessions? - protected void updateGroup(String group_id, String group_name, String peer_id, boolean is_in, boolean is_bot, String text){ + protected void updateGroup(String message_id, String group_id, String group_username, String group_title, String peer_id, boolean is_in, boolean is_bot, String text){ try { //1) get group by id (eg. "telegram_id") String name_id = name+" id"; - String full_group_name = name + ":" + group_name; + String full_group_name = groupFullName(group_title); + String message_link = messageLink(group_username, group_title, group_id, message_id); Collection g = body.storager.getByName(name_id, group_id); Thing group; if (!AL.empty(g)){ @@ -126,7 +136,7 @@ protected void updateGroup(String group_id, String group_name, String peer_id, b group.setString(name_id,group_id); group.storeNew(body.storager); } -body.debug(Writer.capitalize(name)+" channel name_id "+name_id+" group_name "+group_name+" group "+group.toString()+" peer_id "+peer_id+" text "+text);//TODO: remove debug +body.debug(Writer.capitalize(name)+" channel name_id "+name_id+" name"+group_username+" title "+group_title+" group "+group.toString()+" peer_id "+peer_id+" text "+text);//TODO: remove debug //3) if (!is_bot), add/remove peer to/from group if (!is_bot){//TODO: handle bots as well!? //4) get peer by id (eg. "telegram_id") @@ -163,7 +173,7 @@ protected void updateGroup(String group_id, String group_name, String peer_id, b Date today = Time.today(0); Iter parse = new Iter(Parser.parse(text)); if (!AL.empty(topics)) for (Iterator tit = topics.iterator(); tit.hasNext();) - matcher.match(parse, null, (Thing)tit.next(), today, full_group_name, null, thingPaths, null, null, null); + matcher.match(parse, null, (Thing)tit.next(), today, message_link, null, thingPaths, null, null, null); //8) send update if topic is matched //TODO: exclude sender in the news update body.getPublisher().update(null,today,thingPaths,true,group);//forced diff --git a/src/main/java/net/webstructor/comm/Slacker.java b/src/main/java/net/webstructor/comm/Slacker.java index a88ee55..1825904 100644 --- a/src/main/java/net/webstructor/comm/Slacker.java +++ b/src/main/java/net/webstructor/comm/Slacker.java @@ -215,7 +215,8 @@ public boolean handleHTTP(HTTPeer parent, String url, String header, String requ String channel_name = groupName(channel); //TODO:2) check if user is bot boolean is_bot = false;//TODO check if is bot - updateGroup(channel, channel_name, user, true, is_bot, text); +//TODO: message_id + updateGroup(null, channel, channel_name, channel_name, user, true, is_bot, text); //TODO: be able to do unauthorized group conversations and enable chat message handling! diff --git a/src/main/java/net/webstructor/comm/Socializer.java b/src/main/java/net/webstructor/comm/Socializer.java index 4337ba1..e1b27de 100644 --- a/src/main/java/net/webstructor/comm/Socializer.java +++ b/src/main/java/net/webstructor/comm/Socializer.java @@ -382,6 +382,29 @@ public final String cacheReport(SocialFeeder feeder, HashMap feeds, String user_ my_posts_for_the_period = "my posts for the period", reputation = "reputation in community", social_graph = "social graph"; +/* +? ``all connections`` - all other users connected my means of communications to the current user **(cross peers != null)** +- ``my interests`` - **default**, clusters of the posts/messages corresponding to interests of the current user, labeled by keywords typical to respective posts/messages +- ``interests of my friends`` - **default**, clusters of the posts/messages corresponding to interests of other users excluding the current user, labeled by keywords typical to respective posts/messages +- ``similar to me`` - **default**, other users ranked by simlarity in respect to the current user +- ``best friends`` - **default**, other users that are the most involved in mutual communications (likes, votes, comments and mentions) with the current user +- ``fans`` - **default**, other users that are the most involved in communications (likes, votes, comments and mentions) directed to the current user but not the other way around +- ``like and comment me`` - other users who provide likes and votes in respect to the current user +- ``authorities`` - **default**, other users that are getting the most of communications (likes, votes, comments and mentions) from the current user +- ``reputation`` - **default**, list of the users with highest reputation score within entire reachable community, including any users visible given privacy restrictions +- ``social graph`` - **default**, rendering of the nearest social environment in form of interactive social graph +- ``liked by me`` - other users that are getting the most of votes and likes from the current user +- ``my karma by periods`` - dynamic of the "karma" (social capital) for the current user +- ``my words by periods`` - **default**, words used by the current user getting most of attention (likes, votes, comments and mentions) from other users, broken down by time periods +- ``my friends by periods`` - **default**, other users paying attention (likes, votes, comments and mentions) to the current user, broken down by time periods +- ``my favorite words`` - **default**, words most oftenly used, liked and commented by the current user +- ``my posts liked and commented`` - **default**, posts of the current user most oftenly liked and commented by the other users +- ``my best words`` - words most oftenly liked and commented by all of the users including the current user +- ``my words liked and commented`` - words of the current user most oftenly liked and commented by the other users +- ``words liked by me`` - words most oftenly liked by the current user +- ``words of my friends`` - **default**, words most oftenly used by users other than current user +- ``my posts for the period`` - **default**, all posts by the current user for given period +*/ static final String[] report_options = { my_interests, interests_of_my_friends, diff --git a/src/main/java/net/webstructor/comm/telegram/Telegram.java b/src/main/java/net/webstructor/comm/telegram/Telegram.java index 71c544d..a1e2974 100644 --- a/src/main/java/net/webstructor/comm/telegram/Telegram.java +++ b/src/main/java/net/webstructor/comm/telegram/Telegram.java @@ -192,14 +192,16 @@ public void save() { } } - protected void updateInteraction(Date date, String group_id, String group_name, String message_id, String reply_to_message_id, String from_id, String reply_to_from_id, java.util.Set mention_ids, String text_body) { + protected void updateInteraction(Date date, String group_id, String group_name, String message_id, String reply_to_message_id, String from_id, String reply_to_from_id, java.util.Set mention_ids, String text) { if (debug) //TODO: debug body.debug("Telegram crawling from "+from_id+" to "+reply_to_from_id+" mentions "+mention_ids); + text = text.replace('\n', ' '); + String to_id = !AL.empty(reply_to_from_id) ? reply_to_from_id : group_id; - int intvalue = text_body != null ? text_body.length() : 1;//empty comment still counts - String weight = text_body != null ? String.valueOf(intvalue) : null; + int intvalue = text != null ? text.length() : 1;//empty comment still counts + String weight = text != null ? String.valueOf(intvalue) : null; //https://t.me/agirussia/8855 (public - agirussia, 8855) //https://t.me/c/1410910487/75 (private -1001410910487, 75) //String url = !AL.empty(group_name) ? "https://t.me/" + group_name + "/" + message_id : null; @@ -208,9 +210,9 @@ protected void updateInteraction(Date date, String group_id, String group_name, //comments //mentions - int logvalue = 1 + (AL.empty(text_body) ? 0 : (int)Math.round(Math.log10(text_body.length()))); + int logvalue = 1 + (AL.empty(text) ? 0 : (int)Math.round(Math.log10(text.length()))); if (logger != null) - SocialCacher.write(logger, name, date, date.getTime(), "comment", from_id, to_id, weight, null, permlink, parent_permlink, group_name/*title*/, text_body, null, null); + SocialCacher.write(logger, name, date, date.getTime(), "comment", from_id, to_id, weight, null, permlink, parent_permlink, group_name/*title*/, text, null, null); if (!AL.empty(reply_to_from_id)) { updateInteraction(date,"comments",from_id,reply_to_from_id,logvalue);//update from->reply_to_from } diff --git a/src/main/java/net/webstructor/comm/telegram/Telegrammer.java b/src/main/java/net/webstructor/comm/telegram/Telegrammer.java index 5996463..3a675d5 100644 --- a/src/main/java/net/webstructor/comm/telegram/Telegrammer.java +++ b/src/main/java/net/webstructor/comm/telegram/Telegrammer.java @@ -253,6 +253,22 @@ private void delete(String chat_id, String msg_id) throws IOException { body.error("Telegram error",e); } } + + @Override + protected String messageLink(String group_usename, String group_title, String group_id, String message_id) { + //https://stackoverflow.com/questions/51065460/link-message-by-message-id-via-telegram-bot + if (group_usename != null) { + //https://t.me/agirussia/8855 (public - agirussia, 8855) + return "https://t.me/"+group_usename+"/"+message_id; + } + if (group_id.startsWith("-100")) { + //https://t.me/c/1410910487/75 (private -1001410910487, 75) + group_id = group_id.substring(3); + return "https://t.me/c/"+group_id+"/"+message_id; + } + //if not group "type":"supergroup" !? + return super.messageLink(group_usename, group_title, group_id, message_id); + } private long handle(String response) throws IOException{ String botname = body.getSelf().getString(Body.telegram_name); @@ -355,7 +371,7 @@ private long handle(String response) throws IOException{ if (!from_id.equals(chat_id)){ boolean is_bot = from.containsKey("is_bot")? from.getBoolean("is_bot") : false; - updateGroup(chat_id, chat_title, from_id, true, is_bot, text); + updateGroup(message_id, chat_id, chat_username, chat_title, from_id, true, is_bot, text); //process group interactions if (telegram != null) { diff --git a/src/main/java/net/webstructor/comm/twitter/Twitter.java b/src/main/java/net/webstructor/comm/twitter/Twitter.java index dbc6cf0..9ed41a0 100644 --- a/src/main/java/net/webstructor/comm/twitter/Twitter.java +++ b/src/main/java/net/webstructor/comm/twitter/Twitter.java @@ -70,7 +70,7 @@ public TwitterFeeder(Environment body, Twitter api, String user_id, LangPack lan this.api = api; } public void getFeed(String id, String token, String token_secret, Date since, Date until, StringBuilder detail) throws IOException { - body.debug("Twitter crawling "+user_id); + body.debug("Twitter crawling "+id); //https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines //https://developer.twitter.com/en/docs/tweets/timelines/api-reference/get-statuses-home_timeline @@ -81,7 +81,20 @@ public void getFeed(String id, String token, String token_secret, Date since, Da try { //TODO deal with rate limits, if needed //TODO if date limit is not hit, iterate with max_id-1 - String[][] params = new String[][] { + + String[][] params = !Str.isLong(id) ? + new String[][] { + //new String [] {"user_id","563516882"}, + new String [] {"screen_name",id}, + new String [] {"count",String.valueOf(count)}, + //new String [] {"since_id","false"}, + //new String [] {"max_id","1257053173916659712"}, + new String [] {"tweet_mode", "extended"}, + new String [] {"trim_user","false"}, + new String [] {"exclude_replies","false"}, + new String [] {"include_rts","true"} + }: + new String[][] { //new String [] {"user_id","563516882"}, //new String [] {"screen_name","bengoertzel"}, new String [] {"count",String.valueOf(count)}, diff --git a/src/main/java/net/webstructor/core/Property.java b/src/main/java/net/webstructor/core/Property.java index 2385db1..cc8bdbe 100644 --- a/src/main/java/net/webstructor/core/Property.java +++ b/src/main/java/net/webstructor/core/Property.java @@ -23,6 +23,7 @@ */ package net.webstructor.core; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -53,7 +54,8 @@ public class Property extends Anything implements Named { protected String name;//TODO: is protected Anything owner; protected Storager storager = null; - private boolean hasPatterns = false; + private ArrayList patterns = null; + private int limit = DEFAULT_PROPERTY_LIMIT; public Property(Anything owner,String name,String def) { @@ -98,27 +100,38 @@ public String name() { private static final Pattern number = Pattern.compile("[-+−]?[0-9]+([\\.,][0-9]+)?"); public boolean hasPatterns() { - return hasPatterns; + return patterns != null; } - //TODO: build pre-compiled ordered list of patterns and keep in variable!? private void compile() { if (storager == null) return; Thing variable = storager.getThing(name); - if (variable == null) - return; - if (!AL.empty(variable.getThings(AL.patterns))){ - hasPatterns = true; - return; - } - Collection is = variable.getThings(AL.is); - if (!AL.empty(is)) - for (Iterator i = is.iterator(); i.hasNext();) - if (!AL.empty(((Thing)i.next()).getThings(AL.patterns))){ - hasPatterns = true; - return; + compile(variable); + } + + private boolean compile(Thing domain) { + boolean compiled = false; + if (domain != null) { + Collection ps = domain.getThings(AL.patterns); + if (!AL.empty(ps)) for (Object p : ps) { + Thing t = (Thing)p; + String patstr = t.name(); + if (!AL.empty(patstr)) { + if (patterns == null) + patterns = new ArrayList(1); + Set pat = Reader.pat(storager, owner instanceof Thing ? (Thing)owner : null, patstr); + patterns.add(pat); + compiled = true; } + } + Collection is = domain.getThings(AL.is); + if (!AL.empty(is)) + for (Iterator i = is.iterator(); i.hasNext();) + if (compile((Thing)i.next())) + compiled = true; + } + return compiled; } public boolean read(Iter it, StringBuilder summary){ @@ -126,10 +139,7 @@ public boolean read(Iter it, StringBuilder summary){ return false; StringBuilder value = new StringBuilder(); //if variable has patterns, recursively to variable domains - Thing variable = storager.getThing(name); - if (variable == null) - return false; - if (read(it, variable, value)){ + if (readVariable(it, value)){ //TODO: entity extraction goes here!? String v = value.toString(); setString(v); @@ -139,27 +149,16 @@ public boolean read(Iter it, StringBuilder summary){ return false; } - //TODO: move to static Reader, once patterns are pre-compiled - public boolean read(Iter it, Thing domain, StringBuilder summary){ - //first, check patterns of the domain - Collection ps = domain.getThings(AL.patterns); - if (!AL.empty(ps)){ - for (Iterator i = ps.iterator(); i.hasNext();){ - //TODO: system-wide cache of compiled patterns!? - String patstr = ((Thing)i.next()).name(); - Set pat = Reader.pat(storager, owner instanceof Thing ? (Thing)owner : null, patstr); - if (Reader.read(it, pat, summary)) - return true; - } - } - //next, check if domain has super-domains with patterns - Collection domains = (Collection)domain.get(AL.is); - if (!AL.empty(domains)){ - for (Iterator i = domains.iterator(); i.hasNext();){ - if (read(it, (Thing)i.next(), summary)) + private boolean readVariable(Iter it, StringBuilder summary){ + Thing variable = storager.getThing(name); + if (variable == null) + return false; + if (!AL.empty(patterns))//precompiled patterns + for (Set pat : patterns) { + boolean read = Reader.read(it, pat, summary); + if (read) return true; } - } return false; } diff --git a/src/main/java/net/webstructor/data/Emotioner.java b/src/main/java/net/webstructor/data/Emotioner.java index 6f58f1c..2ac3a0d 100644 --- a/src/main/java/net/webstructor/data/Emotioner.java +++ b/src/main/java/net/webstructor/data/Emotioner.java @@ -32,9 +32,9 @@ class Emoticon { //https://lemire.me/blog/2018/06/15/emojis-java-and-strings/ //https://www.branah.com/unicode-converter public class Emotioner { - public static final String positive = "\ud83d\ude0a";//😊0001f60a - public static final String negative = "\ud83d\ude1e";//😞0001f61e - public static final String flushed = "\ud83d\ude33";//😳0001F633 + public static final String positive = "\ud83d\ude0a";//😊0001f60a//smiling face with smiling eyes + public static final String negative = "\ud83d\ude1e";//😞0001f61e//disappointed face + public static final String flushed = "\ud83d\ude33";//😳0001F633//flushed face public static final String emotion(int s) {//sentiment return s < -50 ? Emotioner.negative : s > 50 ? Emotioner.positive : ""; } diff --git a/src/main/java/net/webstructor/peer/Conversation.java b/src/main/java/net/webstructor/peer/Conversation.java index 9eca5c9..2829450 100644 --- a/src/main/java/net/webstructor/peer/Conversation.java +++ b/src/main/java/net/webstructor/peer/Conversation.java @@ -603,7 +603,7 @@ boolean tryReport(Storager storager,Session session) { && session.sessioner.body.getSocializer(arg.getString("network")) != null && arg.getString("id") != null && arg.getString("network") != null //if either a) provider is "public" or b) specified id is matching user id or c) we supply the auth token - && (session.sessioner.body.getSocializer(arg.getString("network")).opendata() || arg.getString("id").equals(session.getStoredPeer().getString(arg.getString("network")+" id")) + && (session.trusted() || session.sessioner.body.getSocializer(arg.getString("network")).opendata() || arg.getString("id").equals(session.getStoredPeer().getString(arg.getString("network")+" id")) || session.read(new Seq(new Object[]{"token",new Property(arg,"token")}))) ) { String format = session.read(new Seq(new Object[]{"format",new Property(arg,"format")})) ? arg.getString("format") : "html"; int threshold = session.read(new Seq(new Object[]{"threshold",new Property(arg,"threshold")})) ? Integer.valueOf(arg.getString("threshold")).intValue() : 20; @@ -617,7 +617,7 @@ boolean tryReport(Storager storager,Session session) { String id = arg.getString("id"); String token = session.read(new Seq(new Object[]{"token",new Property(arg,"token")})) ? arg.getString("token") : null; if (AL.empty(token)) //if token is not supplied explicitly - token = arg.getString("id").equals(session.getStoredPeer().getString(arg.getString("network")+" id")) ? session.getStoredPeer().getString(provider.name()+" token") : null; + token = session.trusted() || arg.getString("id").equals(session.getStoredPeer().getString(arg.getString("network")+" id")) ? session.getStoredPeer().getString(provider.name()+" token") : null; //TODO: name and language for opendata/steemit? String secret = provider.getTokenSecret(session.getStoredPeer()); String report = provider.cachedReport(id,token,secret,id,"",language,format,fresh,session.input(),threshold,period,areas); diff --git a/src/main/java/net/webstructor/peer/Finder.java b/src/main/java/net/webstructor/peer/Finder.java index 3a40789..7528a06 100644 --- a/src/main/java/net/webstructor/peer/Finder.java +++ b/src/main/java/net/webstructor/peer/Finder.java @@ -58,7 +58,8 @@ public String name() { } public boolean searchable(String name){ - return AL.name.equals(name) || !Array.contains(AL.foundation, name); + return AL.name.equals(name) || !(Array.contains(AL.foundation, name) + || AL.sentiment.equals(name));//TODO: exclude it in less hacky way? } @Override diff --git a/src/main/java/net/webstructor/util/Str.java b/src/main/java/net/webstructor/util/Str.java index c690ab2..8aa776e 100644 --- a/src/main/java/net/webstructor/util/Str.java +++ b/src/main/java/net/webstructor/util/Str.java @@ -100,6 +100,17 @@ public static boolean has(String[] args, String name, String expected){ return expected == null || expected.equalsIgnoreCase(val);//any value present or matching expected } + public static boolean isLong(String s) { + if (s == null) + return false; + try { + Long.parseLong(s); + } catch (NumberFormatException nfe) { + return false; + } + return true; + } + public static Object[][] get(String[] args, String[] names, Class[] types){ return get(args, names, types, null); }