Add 'Title resolver' for links

src/main/java/InnaIrcBot/Commanders/ChanelCommander.java

@@ -1,6 +1,5 @@
 package InnaIrcBot.Commanders;
 
-import InnaIrcBot.Commanders.flood.EventHandler;
 import InnaIrcBot.Commanders.flood.JoinCloneHandler;
 import InnaIrcBot.Commanders.flood.JoinFloodHandler;
 import InnaIrcBot.Commanders.talk.TalkGenericHandler;
@@ -8,6 +7,8 @@ import InnaIrcBot.Commanders.talk.TalkHandler;
 import InnaIrcBot.Commanders.talk.TalkZeroHandler;
 import InnaIrcBot.config.ConfigurationChannel;
 import InnaIrcBot.config.ConfigurationManager;
+import InnaIrcBot.linkstitles.LinksTitleManager;
+import InnaIrcBot.linkstitles.LinksTitleRequest;
 
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
@@ -23,6 +24,8 @@ public class ChanelCommander implements Runnable {
     private TalkHandler talkHandler;
     private final List<EventHandler> eventHandlers;
 
+    private BlockingQueue<LinksTitleRequest> urlParser;
+
     public ChanelCommander(BlockingQueue<String> commanderQueue, String server, String channel) throws Exception{
         this.commanderQueue = commanderQueue;
         this.server = server;
@@ -60,6 +63,9 @@ public class ChanelCommander implements Runnable {
                     configChannel.getJoinCloneControlTimeframe());
             eventHandlers.add(jch);
         }
+
+        if (configChannel.isParseLinksTitles())
+            urlParser = LinksTitleManager.getHandlerQueue();
     }
 
     @Override
@@ -97,6 +103,7 @@ public class ChanelCommander implements Runnable {
                 talkHandler.joinCame(dataStrings[0]);
                 break;
             case "PRIVMSG":
+                parseLinks(dataStrings[2]);
                 talkHandler.privmsgCame(dataStrings[0], dataStrings[2]);
                 break;
                 /*  case "PART":
@@ -108,4 +115,10 @@ public class ChanelCommander implements Runnable {
                 break;
         }
     }
+
+    private void parseLinks(String message){
+        if (urlParser != null) {
+            urlParser.add(new LinksTitleRequest(server, channel, message));
+        }
+    }
 }

src/main/java/InnaIrcBot/Commanders/EventHandler.java

@@ -1,4 +1,4 @@
-package InnaIrcBot.Commanders.flood;
+package InnaIrcBot.Commanders;
 
 public interface EventHandler {
     void track(String user);

src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java

@@ -1,5 +1,6 @@
 package InnaIrcBot.Commanders.flood;
 
+import InnaIrcBot.Commanders.EventHandler;
 import InnaIrcBot.ProvidersConsumers.StreamProvider;
 
 import java.time.LocalDateTime;

src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java

@@ -1,5 +1,6 @@
 package InnaIrcBot.Commanders.flood;
 
+import InnaIrcBot.Commanders.EventHandler;
 import InnaIrcBot.ProvidersConsumers.StreamProvider;
 
 import java.time.LocalDateTime;

src/main/java/InnaIrcBot/GlobalData.java

@@ -10,4 +10,5 @@ public class GlobalData {
     }
     public static final String applicationHomePage = "https://github.com/developersu/InnaIrcBot";
     public static final int CHANNEL_QUEUE_CAPACITY = 500;
+    public static final int LINKS_COOLDOWN_FRAME = 3;
 }

src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java

@@ -129,7 +129,7 @@ public class SystemConsumer implements Runnable{
                 IrcChannel ircChannel = channels.get(channelName);
                 if (ircChannel == null)
                     return;
-                ircChannel.getChannelQueue().add(eventNum+" "+sender+" "+message);
+                ircChannel.getChannelQueue().add(sender+" "+eventNum+" "+message);
                 break;
             case "NICK":
                 if (sender.startsWith(nick+"!")) {

src/main/java/InnaIrcBot/config/ConfigurationChannel.java

@@ -18,6 +18,8 @@ public class ConfigurationChannel {
     private int joinCloneControlTimeframe;
     private String joinCloneControlPattern;
 
+    private boolean parseLinksTitles;
+
     public ConfigurationChannel(
             boolean joinFloodControl,
             int joinFloodControlEvents,
@@ -25,7 +27,9 @@ public class ConfigurationChannel {
             boolean joinCloneControl,
             int joinCloneControlTimeframe,
             String joinCloneControlPattern,
-            List<String> rules){
+            boolean parseLinksTitles,
+            List<String> rules)
+    {
 
         parseRules(rules);
 
@@ -34,6 +38,8 @@ public class ConfigurationChannel {
 
         if (joinCloneControl)
             validateJoinCloneControl(joinCloneControlTimeframe, joinCloneControlPattern);
+
+        this.parseLinksTitles = parseLinksTitles;
     }
 
     private void parseRules(List<String> rules){
@@ -99,4 +105,6 @@ public class ConfigurationChannel {
     public boolean isJoinCloneControl() { return joinCloneControl; }
     public int getJoinCloneControlTimeframe() { return joinCloneControlTimeframe; }
     public String getJoinCloneControlPattern() { return joinCloneControlPattern; }
+
+    public boolean isParseLinksTitles() { return parseLinksTitles; }
 }

src/main/java/InnaIrcBot/config/ConfigurationFile.java

@@ -2,6 +2,7 @@ package InnaIrcBot.config;
 
 import org.ini4j.Config;
 import org.ini4j.Ini;
+import org.ini4j.Profile;
 import org.ini4j.Wini;
 
 import java.io.File;
@@ -115,7 +116,8 @@ public class ConfigurationFile {
 
         Ini.Section joinCloneControlSection = channelSection.getChild("rules");
 
-        boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class);;
+        boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class);
+
         int joinCloneControlTimeFrame = -1;
         String joinCloneControlPattern = "";
         if (joinCloneControl){
@@ -123,6 +125,10 @@ public class ConfigurationFile {
             joinCloneControlPattern = joinCloneControlSection.getOrDefault("pattern", "");
         }
 
+        Profile.Section parseLinksTitlesSection = channelSection.getChild("ParseLinksTitles");
+
+        boolean parseLinksTitles = parseLinksTitlesSection.get("enable", boolean.class);
+
         channelConfigs.put(channelName, new ConfigurationChannel(
                 joinFloodControl,
                 joinFloodControlEventsNumber,
@@ -130,6 +136,7 @@ public class ConfigurationFile {
                 joinCloneControl,
                 joinCloneControlTimeFrame,
                 joinCloneControlPattern,
+                parseLinksTitles,
                 channelRules));
     }
 

src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java

@@ -147,6 +147,9 @@ public class ConfigurationFileGenerator {
         channelMainJoinCloneControlSection.put("pattern", "^.+[0-9]+?!.*$");
         channelMainJoinFloodControlSection.put("time frame", 0);
 
+        Ini.Section linksHeaderParser = channelMainSection.addChild("ParseLinksTitles");
+        linksHeaderParser.put("enable", true);
+
         ini.store(configurationFile);
     }
 }

src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java

@@ -0,0 +1,99 @@
+package InnaIrcBot.linkstitles;
+
+import InnaIrcBot.GlobalData;
+import InnaIrcBot.ProvidersConsumers.StreamProvider;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.BlockingQueue;
+// TODO: add timeout to constructor cooldown time etc.
+
+class LinksTitleHandler implements Runnable{
+    private static final int READ_TIMEOUT = 1000;
+
+    private final BlockingQueue<LinksTitleRequest> queue;
+    private LocalDateTime lastReplyTime;
+
+    LinksTitleHandler(BlockingQueue<LinksTitleRequest> queue){
+        this.queue = queue;
+        this.lastReplyTime = LocalDateTime.now();
+    }
+
+    @Override
+    public void run() {
+        System.out.println("["+ LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread started");// TODO:REMOVE DEBUG
+        try {
+            while (true) {
+                LinksTitleRequest request = queue.take();
+                String server = request.getServer();
+                String channel = request.getChannel();
+                String message = request.getMessage();
+
+                if (isTooManyRequests())
+                    continue;
+
+                lastReplyTime = LocalDateTime.now();
+
+                track(server, channel, message);
+            }
+        }
+        catch (InterruptedException ignore){ }
+        System.out.println("["+LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread ended");// TODO:REMOVE DEBUG
+    }
+
+    private void track(String server, String channel, String message) {
+        try {
+            if (! message.contains("http"))
+                return;
+
+            int httpPosition = message.indexOf("http"); // TODO: fix http:// g asdasd https://sadasd.com/
+            String link = message.substring(httpPosition).replaceAll("\\s.+$", "");
+
+            URL url = new URL(link);
+            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection.setReadTimeout(READ_TIMEOUT);
+            int responseCode = connection.getResponseCode();
+            if (responseCode != 200) {
+                //System.out.println("reply "+connection.getResponseCode());
+                return;
+            }
+            if (! connection.getContentType().contains("text/html")){
+                return;
+            }
+            BufferedReader reader = new BufferedReader(
+                    new InputStreamReader(connection.getInputStream())
+            );
+
+            StringBuilder stringBuffer = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                stringBuffer.append(line);
+                if (line.contains("</title>")) {
+                    break;
+                }
+            }
+            reader.close();
+            connection.disconnect();
+            line = stringBuffer.toString();
+            int from = line.indexOf("<title>")+7;
+            int till = line.indexOf("</title>");
+            String title = line.substring(from, till);
+
+            if (title.length() > 510)
+                title = title.substring(0, 510);
+
+            StreamProvider.writeToStream(server, "PRIVMSG  "+ channel +" :"+title+"\n");
+        } catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    private boolean isTooManyRequests(){
+        return lastReplyTime.isAfter(LocalDateTime.now().minusSeconds(GlobalData.LINKS_COOLDOWN_FRAME));
+    }
+}

src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java

@@ -0,0 +1,18 @@
+package InnaIrcBot.linkstitles;
+
+import InnaIrcBot.GlobalData;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
+public class LinksTitleManager {
+    private static final BlockingQueue<LinksTitleRequest> queue = new ArrayBlockingQueue<>(GlobalData.CHANNEL_QUEUE_CAPACITY);
+    private static final Thread thread = new Thread(new LinksTitleHandler(queue));
+
+    public static synchronized BlockingQueue<LinksTitleRequest> getHandlerQueue(){
+        if (! thread.isAlive()){
+            thread.start();
+        }
+        return queue;
+    }
+}

src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java

@@ -0,0 +1,17 @@
+package InnaIrcBot.linkstitles;
+
+public class LinksTitleRequest {
+    private final String server;
+    private final String channel;
+    private final String message;
+
+    public LinksTitleRequest(String server, String channel, String message){
+        this.server = server;
+        this.channel = channel;
+        this.message = message;
+    }
+
+    public String getServer() { return server; }
+    public String getChannel() { return channel; }
+    public String getMessage() { return message; }
+}

src/main/java/InnaIrcBot/logging/WorkerFiles.java

@@ -66,7 +66,7 @@ public class WorkerFiles implements Worker {
     private void createFileWriter() throws IOException{
         dateOnFile = LocalDate.now();
         File newFile = new File(filePath+channel+"_"+ dateOnFile +".txt");
-        fileWriter = new FileWriter(newFile);
+        fileWriter = new FileWriter(newFile, true);
     }
 
     @Override
@@ -105,6 +105,8 @@ public class WorkerFiles implements Worker {
             case "TOPIC":
                 TOPIC(initiator, message);
                 break;
+            case "353":
+                break;
             default:
                 prettyPrint("["+LocalTime.now().format(dateFormat)+"] "+event+" "+initiator+" "+message+"\n");   // TODO: QA @ big data
                 break;

src/main/java/InnaIrcBot/logging/WorkerMongoDB.java

@@ -147,6 +147,8 @@ public class WorkerMongoDB implements Worker {               //TODO consider ski
             case "PRIVMSG":
                 document.append("message1", message.replaceAll("^:", ""));
                 break;
+            case "353":
+                break;
             case "NICK":
             case "JOIN":
             default:

src/main/java/InnaIrcBot/logging/WorkerSQLite.java

@@ -174,6 +174,8 @@ public class WorkerSQLite implements Worker {
                     preparedStatement.setString(4, message.replaceAll("^:", ""));
                     preparedStatement.setString(5,null);
                     break;
+                case "353":
+                    break;
                 case "NICK":
                 case "JOIN":
                 default: