desu
/
innaircbot
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add 'Title resolver' for links

master
Dmitry Isaenko 2020-11-01 18:35:14 +03:00
parent 2c9760785e
commit 0197f00250
15 changed files with 180 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/InnaIrcBot/Commanders/ChanelCommander.java
View File

@ -1,6 +1,5 @@
package InnaIrcBot.Commanders;
import InnaIrcBot.Commanders.flood.EventHandler;
import InnaIrcBot.Commanders.flood.JoinCloneHandler;
import InnaIrcBot.Commanders.flood.JoinFloodHandler;
import InnaIrcBot.Commanders.talk.TalkGenericHandler;
@ -8,6 +7,8 @@ import InnaIrcBot.Commanders.talk.TalkHandler;
import InnaIrcBot.Commanders.talk.TalkZeroHandler;
import InnaIrcBot.config.ConfigurationChannel;
import InnaIrcBot.config.ConfigurationManager;
import InnaIrcBot.linkstitles.LinksTitleManager;
import InnaIrcBot.linkstitles.LinksTitleRequest;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
@ -23,6 +24,8 @@ public class ChanelCommander implements Runnable {
private TalkHandler talkHandler;
private final List<EventHandler> eventHandlers;
private BlockingQueue<LinksTitleRequest> urlParser;
public ChanelCommander(BlockingQueue<String> commanderQueue, String server, String channel) throws Exception{
this.commanderQueue = commanderQueue;
this.server = server;
@ -60,6 +63,9 @@ public class ChanelCommander implements Runnable {
configChannel.getJoinCloneControlTimeframe());
eventHandlers.add(jch);
}
if (configChannel.isParseLinksTitles())
urlParser = LinksTitleManager.getHandlerQueue();
}
@Override
@ -97,6 +103,7 @@ public class ChanelCommander implements Runnable {
talkHandler.joinCame(dataStrings[0]);
break;
case "PRIVMSG":
parseLinks(dataStrings[2]);
talkHandler.privmsgCame(dataStrings[0], dataStrings[2]);
break;
/* case "PART":
@ -108,4 +115,10 @@ public class ChanelCommander implements Runnable {
break;
}
}
private void parseLinks(String message){
if (urlParser != null) {
urlParser.add(new LinksTitleRequest(server, channel, message));
}
}
}

2
src/main/java/InnaIrcBot/Commanders/flood/EventHandler.java → src/main/java/InnaIrcBot/Commanders/EventHandler.java
View File

@ -1,4 +1,4 @@
package InnaIrcBot.Commanders.flood;
package InnaIrcBot.Commanders;
public interface EventHandler {
void track(String user);

1
src/main/java/InnaIrcBot/Commanders/flood/JoinCloneHandler.java
View File

@ -1,5 +1,6 @@
package InnaIrcBot.Commanders.flood;
import InnaIrcBot.Commanders.EventHandler;
import InnaIrcBot.ProvidersConsumers.StreamProvider;
import java.time.LocalDateTime;

1
src/main/java/InnaIrcBot/Commanders/flood/JoinFloodHandler.java
View File

@ -1,5 +1,6 @@
package InnaIrcBot.Commanders.flood;
import InnaIrcBot.Commanders.EventHandler;
import InnaIrcBot.ProvidersConsumers.StreamProvider;
import java.time.LocalDateTime;

1
src/main/java/InnaIrcBot/GlobalData.java
View File

@ -10,4 +10,5 @@ public class GlobalData {
}
public static final String applicationHomePage = "https://github.com/developersu/InnaIrcBot";
public static final int CHANNEL_QUEUE_CAPACITY = 500;
public static final int LINKS_COOLDOWN_FRAME = 3;
}

2
src/main/java/InnaIrcBot/ProvidersConsumers/SystemConsumer.java
View File

@ -129,7 +129,7 @@ public class SystemConsumer implements Runnable{
IrcChannel ircChannel = channels.get(channelName);
if (ircChannel == null)
return;
ircChannel.getChannelQueue().add(eventNum+" "+sender+" "+message);
ircChannel.getChannelQueue().add(sender+" "+eventNum+" "+message);
break;
case "NICK":
if (sender.startsWith(nick+"!")) {

10
src/main/java/InnaIrcBot/config/ConfigurationChannel.java
View File

@ -18,6 +18,8 @@ public class ConfigurationChannel {
private int joinCloneControlTimeframe;
private String joinCloneControlPattern;
private boolean parseLinksTitles;
public ConfigurationChannel(
boolean joinFloodControl,
int joinFloodControlEvents,
@ -25,7 +27,9 @@ public class ConfigurationChannel {
boolean joinCloneControl,
int joinCloneControlTimeframe,
String joinCloneControlPattern,
List<String> rules){
boolean parseLinksTitles,
List<String> rules)
{
parseRules(rules);
@ -34,6 +38,8 @@ public class ConfigurationChannel {
if (joinCloneControl)
validateJoinCloneControl(joinCloneControlTimeframe, joinCloneControlPattern);
this.parseLinksTitles = parseLinksTitles;
}
private void parseRules(List<String> rules){
@ -99,4 +105,6 @@ public class ConfigurationChannel {
public boolean isJoinCloneControl() { return joinCloneControl; }
public int getJoinCloneControlTimeframe() { return joinCloneControlTimeframe; }
public String getJoinCloneControlPattern() { return joinCloneControlPattern; }
public boolean isParseLinksTitles() { return parseLinksTitles; }
}

9
src/main/java/InnaIrcBot/config/ConfigurationFile.java
View File

@ -2,6 +2,7 @@ package InnaIrcBot.config;
import org.ini4j.Config;
import org.ini4j.Ini;
import org.ini4j.Profile;
import org.ini4j.Wini;
import java.io.File;
@ -115,7 +116,8 @@ public class ConfigurationFile {
Ini.Section joinCloneControlSection = channelSection.getChild("rules");
boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class);;
boolean joinCloneControl = joinCloneControlSection.get("enable", boolean.class);
int joinCloneControlTimeFrame = -1;
String joinCloneControlPattern = "";
if (joinCloneControl){
@ -123,6 +125,10 @@ public class ConfigurationFile {
joinCloneControlPattern = joinCloneControlSection.getOrDefault("pattern", "");
}
Profile.Section parseLinksTitlesSection = channelSection.getChild("ParseLinksTitles");
boolean parseLinksTitles = parseLinksTitlesSection.get("enable", boolean.class);
channelConfigs.put(channelName, new ConfigurationChannel(
joinFloodControl,
joinFloodControlEventsNumber,
@ -130,6 +136,7 @@ public class ConfigurationFile {
joinCloneControl,
joinCloneControlTimeFrame,
joinCloneControlPattern,
parseLinksTitles,
channelRules));
}

3
src/main/java/InnaIrcBot/config/ConfigurationFileGenerator.java
View File

@ -147,6 +147,9 @@ public class ConfigurationFileGenerator {
channelMainJoinCloneControlSection.put("pattern", "^.+[0-9]+?!.*$");
channelMainJoinFloodControlSection.put("time frame", 0);
Ini.Section linksHeaderParser = channelMainSection.addChild("ParseLinksTitles");
linksHeaderParser.put("enable", true);
ini.store(configurationFile);
}
}

99
src/main/java/InnaIrcBot/linkstitles/LinksTitleHandler.java Normal file
View File

@ -0,0 +1,99 @@
package InnaIrcBot.linkstitles;
import InnaIrcBot.GlobalData;
import InnaIrcBot.ProvidersConsumers.StreamProvider;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.BlockingQueue;
// TODO: add timeout to constructor cooldown time etc.
class LinksTitleHandler implements Runnable{
private static final int READ_TIMEOUT = 1000;
private final BlockingQueue<LinksTitleRequest> queue;
private LocalDateTime lastReplyTime;
LinksTitleHandler(BlockingQueue<LinksTitleRequest> queue){
this.queue = queue;
this.lastReplyTime = LocalDateTime.now();
}
@Override
public void run() {
System.out.println("["+ LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread started");// TODO:REMOVE DEBUG
try {
while (true) {
LinksTitleRequest request = queue.take();
String server = request.getServer();
String channel = request.getChannel();
String message = request.getMessage();
if (isTooManyRequests())
continue;
lastReplyTime = LocalDateTime.now();
track(server, channel, message);
}
}
catch (InterruptedException ignore){ }
System.out.println("["+LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"))+"] LinksTitleHandler thread ended");// TODO:REMOVE DEBUG
}
private void track(String server, String channel, String message) {
try {
if (! message.contains("http"))
return;
int httpPosition = message.indexOf("http"); // TODO: fix http:// g asdasd https://sadasd.com/
String link = message.substring(httpPosition).replaceAll("\\s.+$", "");
URL url = new URL(link);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setReadTimeout(READ_TIMEOUT);
int responseCode = connection.getResponseCode();
if (responseCode != 200) {
//System.out.println("reply "+connection.getResponseCode());
return;
}
if (! connection.getContentType().contains("text/html")){
return;
}
BufferedReader reader = new BufferedReader(
new InputStreamReader(connection.getInputStream())
);
StringBuilder stringBuffer = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
stringBuffer.append(line);
if (line.contains("</title>")) {
break;
}
}
reader.close();
connection.disconnect();
line = stringBuffer.toString();
int from = line.indexOf("<title>")+7;
int till = line.indexOf("</title>");
String title = line.substring(from, till);
if (title.length() > 510)
title = title.substring(0, 510);
StreamProvider.writeToStream(server, "PRIVMSG "+ channel +" :"+title+"\n");
} catch (Exception e){
e.printStackTrace();
}
}
private boolean isTooManyRequests(){
return lastReplyTime.isAfter(LocalDateTime.now().minusSeconds(GlobalData.LINKS_COOLDOWN_FRAME));
}
}

18
src/main/java/InnaIrcBot/linkstitles/LinksTitleManager.java Normal file
View File

@ -0,0 +1,18 @@
package InnaIrcBot.linkstitles;
import InnaIrcBot.GlobalData;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
public class LinksTitleManager {
private static final BlockingQueue<LinksTitleRequest> queue = new ArrayBlockingQueue<>(GlobalData.CHANNEL_QUEUE_CAPACITY);
private static final Thread thread = new Thread(new LinksTitleHandler(queue));
public static synchronized BlockingQueue<LinksTitleRequest> getHandlerQueue(){
if (! thread.isAlive()){
thread.start();
}
return queue;
}
}

17
src/main/java/InnaIrcBot/linkstitles/LinksTitleRequest.java Normal file
View File

@ -0,0 +1,17 @@
package InnaIrcBot.linkstitles;
public class LinksTitleRequest {
private final String server;
private final String channel;
private final String message;
public LinksTitleRequest(String server, String channel, String message){
this.server = server;
this.channel = channel;
this.message = message;
}
public String getServer() { return server; }
public String getChannel() { return channel; }
public String getMessage() { return message; }
}

4
src/main/java/InnaIrcBot/logging/WorkerFiles.java
View File

@ -66,7 +66,7 @@ public class WorkerFiles implements Worker {
private void createFileWriter() throws IOException{
dateOnFile = LocalDate.now();
File newFile = new File(filePath+channel+"_"+ dateOnFile +".txt");
fileWriter = new FileWriter(newFile);
fileWriter = new FileWriter(newFile, true);
}
@Override
@ -105,6 +105,8 @@ public class WorkerFiles implements Worker {
case "TOPIC":
TOPIC(initiator, message);
break;
case "353":
break;
default:
prettyPrint("["+LocalTime.now().format(dateFormat)+"] "+event+" "+initiator+" "+message+"\n"); // TODO: QA @ big data
break;

2
src/main/java/InnaIrcBot/logging/WorkerMongoDB.java
View File

@ -147,6 +147,8 @@ public class WorkerMongoDB implements Worker { //TODO consider ski
case "PRIVMSG":
document.append("message1", message.replaceAll("^:", ""));
break;
case "353":
break;
case "NICK":
case "JOIN":
default:

2
src/main/java/InnaIrcBot/logging/WorkerSQLite.java
View File

@ -174,6 +174,8 @@ public class WorkerSQLite implements Worker {
preparedStatement.setString(4, message.replaceAll("^:", ""));
preparedStatement.setString(5,null);
break;
case "353":
break;
case "NICK":
case "JOIN":
default: