From 0197f00250758b70f1ca073ec52b29bc836054cc Mon Sep 17 00:00:00 2001 From: Dmitry Isaenko Date: Sun, 1 Nov 2020 18:35:14 +0300 Subject: [PATCH] Add 'Title resolver' for links --- .../Commanders/ChanelCommander.java | 15 ++- .../Commanders/{flood => }/EventHandler.java | 2 +- .../Commanders/flood/JoinCloneHandler.java | 1 + .../Commanders/flood/JoinFloodHandler.java | 1 + src/main/java/InnaIrcBot/GlobalData.java | 1 + .../ProvidersConsumers/SystemConsumer.java | 2 +- .../config/ConfigurationChannel.java | 10 +- .../InnaIrcBot/config/ConfigurationFile.java | 9 +- .../config/ConfigurationFileGenerator.java | 3 + .../linkstitles/LinksTitleHandler.java | 99 +++++++++++++++++++ .../linkstitles/LinksTitleManager.java | 18 ++++ .../linkstitles/LinksTitleRequest.java | 17 ++++ .../java/InnaIrcBot/logging/WorkerFiles.java | 4 +- .../InnaIrcBot/logging/WorkerMongoDB.java | 2 + .../java/InnaIrcBot/logging/WorkerSQLite.java | 2 + 15 files changed, 180 insertions(+), 6 deletions(-) rename src/main/java/InnaIrcBot/Commanders/{flood => }/EventHandler.java (63%) create mode 100644 src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java create mode 100644 src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java create mode 100644 src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java diff --git a/src/main/java/InnaIrcBot/Commanders/ChanelCommander.java b/src/main/java/InnaIrcBot/Commanders/ChanelCommander.java index f446a62..d4e597c 100644 --- a/src/main/java/InnaIrcBot/Commanders/ChanelCommander.java +++ b/src/main/java/InnaIrcBot/Commanders/ChanelCommander.java @@ -1,6 +1,5 @@ package InnaIrcBot.Commanders; -import InnaIrcBot.Commanders.flood.EventHandler; import InnaIrcBot.Commanders.flood.JoinCloneHandler; import InnaIrcBot.Commanders.flood.JoinFloodHandler; import InnaIrcBot.Commanders.talk.TalkGenericHandler; @@ -8,6 +7,8 @@ import InnaIrcBot.Commanders.talk.TalkHandler; import InnaIrcBot.Commanders.talk.TalkZeroHandler; import InnaIrcBot.config.ConfigurationChannel; import InnaIrcBot.config.ConfigurationManager; +import InnaIrcBot.linkstitles.LinksTitleManager; +import InnaIrcBot.linkstitles.LinksTitleRequest; import java.time.LocalTime; import java.time.format.DateTimeFormatter; @@ -23,6 +24,8 @@ public class ChanelCommander implements Runnable { private TalkHandler talkHandler; private final List eventHandlers; + private BlockingQueue urlParser; + public ChanelCommander(BlockingQueue commanderQueue, String server, String channel) throws Exception{ this.commanderQueue = commanderQueue; this.server = server; @@ -60,6 +63,9 @@ public class ChanelCommander implements Runnable { configChannel.getJoinCloneControlTimeframe()); eventHandlers.add(jch); } + + if (configChannel.isParseLinksTitles()) + urlParser = LinksTitleManager.getHandlerQueue(); } @Override @@ -97,6 +103,7 @@ public class ChanelCommander implements Runnable { talkHandler.joinCame(dataStrings[0]); break; case "PRIVMSG": + parseLinks(dataStrings[2]); talkHandler.privmsgCame(dataStrings[0], dataStrings[2]); break; /* case "PART": @@ -108,4 +115,10 @@ public class ChanelCommander implements Runnable { break; } } + + private void parseLinks(String message){ + if (urlParser != null) { + urlParser.add(new LinksTitleRequest(server, channel, message)); + } + } } \ No newline at end of file diff --git a/src/main/java/InnaIrcBot/Commanders/flood/EventHandler.java b/src/main/java/InnaIrcBot/Commanders/EventHandler.java similarity index 63% rename from src/main/java/InnaIrcBot/Commanders/flood/EventHandler.java rename to src/main/java/InnaIrcBot/Commanders/EventHandler.java index f8969ff..c6cc1ad 100644 --- a/src/main/java/InnaIrcBot/Commanders/flood/EventHandler.java +++ b/src/main/java/InnaIrcBot/Commanders/EventHandler.java @@ -1,4 +1,4 @@ -package InnaIrcBot.Commanders.flood; +package InnaIrcBot.Commanders; public interface EventHandler { void track(String user); diff --git a/src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java b/src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java index a6497db..c7ff418 100644 --- a/src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java +++ b/src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java @@ -1,5 +1,6 @@ package InnaIrcBot.Commanders.flood; +import InnaIrcBot.Commanders.EventHandler; import InnaIrcBot.ProvidersConsumers.StreamProvider; import java.time.LocalDateTime; diff --git a/src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java b/src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java index 47e6f03..53046b2 100644 --- a/src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java +++ b/src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java @@ -1,5 +1,6 @@ package InnaIrcBot.Commanders.flood; +import InnaIrcBot.Commanders.EventHandler; import InnaIrcBot.ProvidersConsumers.StreamProvider; import java.time.LocalDateTime; diff --git a/src/main/java/InnaIrcBot/GlobalData.java b/src/main/java/InnaIrcBot/GlobalData.java index 1bac000..b365a69 100644 --- a/src/main/java/InnaIrcBot/GlobalData.java +++ b/src/main/java/InnaIrcBot/GlobalData.java @@ -10,4 +10,5 @@ public class GlobalData { } public static final String applicationHomePage = "https://github.com/developersu/InnaIrcBot"; public static final int CHANNEL_QUEUE_CAPACITY = 500; + public static final int LINKS_COOLDOWN_FRAME = 3; } diff --git a/src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java b/src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java index 45dc010..c9196b4 100644 --- a/src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java +++ b/src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java @@ -129,7 +129,7 @@ public class SystemConsumer implements Runnable{ IrcChannel ircChannel = channels.get(channelName); if (ircChannel == null) return; - ircChannel.getChannelQueue().add(eventNum+" "+sender+" "+message); + ircChannel.getChannelQueue().add(sender+" "+eventNum+" "+message); break; case "NICK": if (sender.startsWith(nick+"!")) { diff --git a/src/main/java/InnaIrcBot/config/ConfigurationChannel.java b/src/main/java/InnaIrcBot/config/ConfigurationChannel.java index 3c467ff..110f933 100644 --- a/src/main/java/InnaIrcBot/config/ConfigurationChannel.java +++ b/src/main/java/InnaIrcBot/config/ConfigurationChannel.java @@ -18,6 +18,8 @@ public class ConfigurationChannel { private int joinCloneControlTimeframe; private String joinCloneControlPattern; + private boolean parseLinksTitles; + public ConfigurationChannel( boolean joinFloodControl, int joinFloodControlEvents, @@ -25,7 +27,9 @@ public class ConfigurationChannel { boolean joinCloneControl, int joinCloneControlTimeframe, String joinCloneControlPattern, - List rules){ + boolean parseLinksTitles, + List rules) + { parseRules(rules); @@ -34,6 +38,8 @@ public class ConfigurationChannel { if (joinCloneControl) validateJoinCloneControl(joinCloneControlTimeframe, joinCloneControlPattern); + + this.parseLinksTitles = parseLinksTitles; } private void parseRules(List rules){ @@ -99,4 +105,6 @@ public class ConfigurationChannel { public boolean isJoinCloneControl() { return joinCloneControl; } public int getJoinCloneControlTimeframe() { return joinCloneControlTimeframe; } public String getJoinCloneControlPattern() { return joinCloneControlPattern; } + + public boolean isParseLinksTitles() { return parseLinksTitles; } } diff --git a/src/main/java/InnaIrcBot/config/ConfigurationFile.java b/src/main/java/InnaIrcBot/config/ConfigurationFile.java index 07814e0..ff34567 100644 --- a/src/main/java/InnaIrcBot/config/ConfigurationFile.java +++ b/src/main/java/InnaIrcBot/config/ConfigurationFile.java @@ -2,6 +2,7 @@ package InnaIrcBot.config; import org.ini4j.Config; import org.ini4j.Ini; +import org.ini4j.Profile; import org.ini4j.Wini; import java.io.File; @@ -115,7 +116,8 @@ public class ConfigurationFile { Ini.Section joinCloneControlSection = channelSection.getChild("rules"); - boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class);; + boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class); + int joinCloneControlTimeFrame = -1; String joinCloneControlPattern = ""; if (joinCloneControl){ @@ -123,6 +125,10 @@ public class ConfigurationFile { joinCloneControlPattern = joinCloneControlSection.getOrDefault("pattern", ""); } + Profile.Section parseLinksTitlesSection = channelSection.getChild("ParseLinksTitles"); + + boolean parseLinksTitles = parseLinksTitlesSection.get("enable", boolean.class); + channelConfigs.put(channelName, new ConfigurationChannel( joinFloodControl, joinFloodControlEventsNumber, @@ -130,6 +136,7 @@ public class ConfigurationFile { joinCloneControl, joinCloneControlTimeFrame, joinCloneControlPattern, + parseLinksTitles, channelRules)); } diff --git a/src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java b/src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java index 3d0f5e9..f0d956b 100644 --- a/src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java +++ b/src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java @@ -147,6 +147,9 @@ public class ConfigurationFileGenerator { channelMainJoinCloneControlSection.put("pattern", "^.+[0-9]+?!.*$"); channelMainJoinFloodControlSection.put("time frame", 0); + Ini.Section linksHeaderParser = channelMainSection.addChild("ParseLinksTitles"); + linksHeaderParser.put("enable", true); + ini.store(configurationFile); } } diff --git a/src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java b/src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java new file mode 100644 index 0000000..d072787 --- /dev/null +++ b/src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java @@ -0,0 +1,99 @@ +package InnaIrcBot.linkstitles; + +import InnaIrcBot.GlobalData; +import InnaIrcBot.ProvidersConsumers.StreamProvider; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.BlockingQueue; +// TODO: add timeout to constructor cooldown time etc. + +class LinksTitleHandler implements Runnable{ + private static final int READ_TIMEOUT = 1000; + + private final BlockingQueue queue; + private LocalDateTime lastReplyTime; + + LinksTitleHandler(BlockingQueue queue){ + this.queue = queue; + this.lastReplyTime = LocalDateTime.now(); + } + + @Override + public void run() { + System.out.println("["+ LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread started");// TODO:REMOVE DEBUG + try { + while (true) { + LinksTitleRequest request = queue.take(); + String server = request.getServer(); + String channel = request.getChannel(); + String message = request.getMessage(); + + if (isTooManyRequests()) + continue; + + lastReplyTime = LocalDateTime.now(); + + track(server, channel, message); + } + } + catch (InterruptedException ignore){ } + System.out.println("["+LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread ended");// TODO:REMOVE DEBUG + } + + private void track(String server, String channel, String message) { + try { + if (! message.contains("http")) + return; + + int httpPosition = message.indexOf("http"); // TODO: fix http:// g asdasd https://sadasd.com/ + String link = message.substring(httpPosition).replaceAll("\\s.+$", ""); + + URL url = new URL(link); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setReadTimeout(READ_TIMEOUT); + int responseCode = connection.getResponseCode(); + if (responseCode != 200) { + //System.out.println("reply "+connection.getResponseCode()); + return; + } + if (! connection.getContentType().contains("text/html")){ + return; + } + BufferedReader reader = new BufferedReader( + new InputStreamReader(connection.getInputStream()) + ); + + StringBuilder stringBuffer = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + stringBuffer.append(line); + if (line.contains("")) { + break; + } + } + reader.close(); + connection.disconnect(); + line = stringBuffer.toString(); + int from = line.indexOf("")+7; + int till = line.indexOf(""); + String title = line.substring(from, till); + + if (title.length() > 510) + title = title.substring(0, 510); + + StreamProvider.writeToStream(server, "PRIVMSG "+ channel +" :"+title+"\n"); + } catch (Exception e){ + e.printStackTrace(); + } + } + + private boolean isTooManyRequests(){ + return lastReplyTime.isAfter(LocalDateTime.now().minusSeconds(GlobalData.LINKS_COOLDOWN_FRAME)); + } +} \ No newline at end of file diff --git a/src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java b/src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java new file mode 100644 index 0000000..c317b99 --- /dev/null +++ b/src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java @@ -0,0 +1,18 @@ +package InnaIrcBot.linkstitles; + +import InnaIrcBot.GlobalData; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +public class LinksTitleManager { + private static final BlockingQueue queue = new ArrayBlockingQueue<>(GlobalData.CHANNEL_QUEUE_CAPACITY); + private static final Thread thread = new Thread(new LinksTitleHandler(queue)); + + public static synchronized BlockingQueue getHandlerQueue(){ + if (! thread.isAlive()){ + thread.start(); + } + return queue; + } +} diff --git a/src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java b/src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java new file mode 100644 index 0000000..2bf5e20 --- /dev/null +++ b/src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java @@ -0,0 +1,17 @@ +package InnaIrcBot.linkstitles; + +public class LinksTitleRequest { + private final String server; + private final String channel; + private final String message; + + public LinksTitleRequest(String server, String channel, String message){ + this.server = server; + this.channel = channel; + this.message = message; + } + + public String getServer() { return server; } + public String getChannel() { return channel; } + public String getMessage() { return message; } +} diff --git a/src/main/java/InnaIrcBot/logging/WorkerFiles.java b/src/main/java/InnaIrcBot/logging/WorkerFiles.java index 9352a49..5aa241b 100644 --- a/src/main/java/InnaIrcBot/logging/WorkerFiles.java +++ b/src/main/java/InnaIrcBot/logging/WorkerFiles.java @@ -66,7 +66,7 @@ public class WorkerFiles implements Worker { private void createFileWriter() throws IOException{ dateOnFile = LocalDate.now(); File newFile = new File(filePath+channel+"_"+ dateOnFile +".txt"); - fileWriter = new FileWriter(newFile); + fileWriter = new FileWriter(newFile, true); } @Override @@ -105,6 +105,8 @@ public class WorkerFiles implements Worker { case "TOPIC": TOPIC(initiator, message); break; + case "353": + break; default: prettyPrint("["+LocalTime.now().format(dateFormat)+"] "+event+" "+initiator+" "+message+"\n"); // TODO: QA @ big data break; diff --git a/src/main/java/InnaIrcBot/logging/WorkerMongoDB.java b/src/main/java/InnaIrcBot/logging/WorkerMongoDB.java index af5b955..c1b0582 100644 --- a/src/main/java/InnaIrcBot/logging/WorkerMongoDB.java +++ b/src/main/java/InnaIrcBot/logging/WorkerMongoDB.java @@ -147,6 +147,8 @@ public class WorkerMongoDB implements Worker { //TODO consider ski case "PRIVMSG": document.append("message1", message.replaceAll("^:", "")); break; + case "353": + break; case "NICK": case "JOIN": default: diff --git a/src/main/java/InnaIrcBot/logging/WorkerSQLite.java b/src/main/java/InnaIrcBot/logging/WorkerSQLite.java index 45fc082..32b26ed 100644 --- a/src/main/java/InnaIrcBot/logging/WorkerSQLite.java +++ b/src/main/java/InnaIrcBot/logging/WorkerSQLite.java @@ -174,6 +174,8 @@ public class WorkerSQLite implements Worker { preparedStatement.setString(4, message.replaceAll("^:", "")); preparedStatement.setString(5,null); break; + case "353": + break; case "NICK": case "JOIN": default: