Merge remote-tracking branch 'origin/development'

FredBoat/pom.xml

@@ -38,9 +38,9 @@
     </properties>
     <dependencies>
         <dependency>
-            <groupId>net.dv8tion</groupId>
+            <groupId>com.github.DV8FromTheWorld</groupId>
             <artifactId>JDA</artifactId>
-            <version>3.1.1_210</version>
+            <version>1a13578</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -115,7 +115,7 @@
         <dependency>
             <groupId>it.unimi.dsi</groupId>
             <artifactId>fastutil</artifactId>
-            <version>7.2.0</version>
+            <version>8.1.0</version>
         </dependency>
 
         <!-- Postgres/Hibernate/OAuth2 stuff -->

FredBoat/src/main/java/fredboat/Config.java

@@ -27,7 +27,7 @@
 
 import com.mashape.unirest.http.exceptions.UnirestException;
 import fredboat.util.DiscordUtil;
-import fredboat.util.DistributionEnum;
+import fredboat.util.constant.DistributionEnum;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -191,7 +191,7 @@ public Config(File credentialsFile, File configFile, int scope) {
 
             //more database connections don't help with performance, so use a value based on available cores
             //http://www.dailymotion.com/video/x2s8uec_oltp-performance-concurrent-mid-tier-connections_tech
-            if (jdbcUrl == null || "".equals(jdbcUrl))
+            if (jdbcUrl == null || "".equals(jdbcUrl) || distribution == DistributionEnum.DEVELOPMENT)
                 //more than one connection for the fallback sqlite db is problematic as there is currently (2017-04-16)
                 // no supported way in the custom driver and/or dialect to set lock timeouts
                 hikariPoolSize = 1;

FredBoat/src/main/java/fredboat/FredBoat.java

@@ -45,16 +45,17 @@
 import fredboat.event.EventListenerSelf;
 import fredboat.event.ShardWatchdogListener;
 import fredboat.feature.I18n;
-import fredboat.util.DistributionEnum;
+import fredboat.util.constant.DistributionEnum;
 import fredboat.util.log.SimpleLogToSLF4JAdapter;
 import frederikam.jca.JCA;
 import frederikam.jca.JCABuilder;
+import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
 import net.dv8tion.jda.core.AccountType;
 import net.dv8tion.jda.core.JDA;
 import net.dv8tion.jda.core.JDAInfo;
 import net.dv8tion.jda.core.entities.Guild;
+import net.dv8tion.jda.core.entities.ISnowflake;
 import net.dv8tion.jda.core.entities.TextChannel;
-import net.dv8tion.jda.core.entities.User;
 import net.dv8tion.jda.core.entities.VoiceChannel;
 import net.dv8tion.jda.core.events.ReadyEvent;
 import net.dv8tion.jda.core.hooks.EventListener;
@@ -67,9 +68,8 @@
 import javax.security.auth.login.LoginException;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -404,16 +404,49 @@ public static List<Guild> getAllGuilds() {
         return list;
     }
 
-    public static Map<String, User> getAllUsersAsMap() {
-        HashMap<String, User> map = new HashMap<>();
+    public static int countAllGuilds() {
+        return Collections.unmodifiableCollection(shards)
+                .stream()
+                .mapToInt(shard -> shard.getJda().getGuilds().size())
+                .sum();
+    }
 
-        for (FredBoat fb : shards) {
-            for (User usr : fb.getJda().getUsers()) {
-                map.put(usr.getId(), usr);
-            }
+    //this probably takes horribly long and should be solved in a different way
+    //rewrite it when we actually come up with a use case for needing all user objects
+//    @Deprecated
+//    public static Map<String, User> getAllUsersAsMap() {
+//        HashMap<String, User> map = new HashMap<>();
+//
+//        for (FredBoat fb : shards) {
+//            for (User usr : fb.getJda().getUsers()) {
+//                map.put(usr.getId(), usr);
+//            }
+//        }
+//        return map;
+//    }
+
+    private static int biggestUserCount = -1;
+
+    //IMPORTANT: do not use this for actually counting, it will not be accurate; it is meant to be used to initialize
+    // sets or maps that are about to hold all those user values
+    public static int getExpectedUserCount() {
+        if (biggestUserCount <= 0) { //initialize
+            countAllUniqueUsers();
         }
+        return biggestUserCount;
+    }
 
-        return map;
+    public static long countAllUniqueUsers() {
+        int expected = biggestUserCount > 0 ? biggestUserCount : LongOpenHashSet.DEFAULT_INITIAL_SIZE;
+        LongOpenHashSet uniqueUsers = new LongOpenHashSet(expected + 100000); //add 100k for good measure
+        Collections.unmodifiableCollection(shards).forEach(
+                shard -> shard.getJda().getUsers().parallelStream().mapToLong(ISnowflake::getIdLong).forEach(uniqueUsers::add)
+        );
+        //never shrink the user count (might happen due to not connected shards)
+        if (uniqueUsers.size() > biggestUserCount) {
+            biggestUserCount = uniqueUsers.size();
+        }
+        return uniqueUsers.size();
     }
 
     public static TextChannel getTextChannelById(String id) {

FredBoat/src/main/java/fredboat/FredBoatBot.java

@@ -106,8 +106,11 @@ public void revive() {
         try {
             channelsToRejoin.clear();
 
-            PlayerRegistry.getPlayingPlayers().stream().filter(guildPlayer -> guildPlayer.getJda().equals(jda))
-                    .forEach(guildPlayer -> channelsToRejoin.add(guildPlayer.getChannel().getId()));
+            PlayerRegistry.getPlayingPlayers().stream()
+                    .filter(guildPlayer -> guildPlayer.getJda().getShardInfo().getShardId() == shardId)
+                    .forEach(guildPlayer -> {
+                        if (guildPlayer.getChannel() != null) channelsToRejoin.add(guildPlayer.getChannel().getId());
+                    });
         } catch (Exception ex) {
             log.error("Caught exception while reviving shard " + this, ex);
         }

FredBoat/src/main/java/fredboat/agent/CarbonitexAgent.java

@@ -26,7 +26,9 @@
 package fredboat.agent;
 
 import com.mashape.unirest.http.Unirest;
+import fredboat.Config;
 import fredboat.FredBoat;
+import net.dv8tion.jda.core.JDA;
 import org.slf4j.LoggerFactory;
 
 public class CarbonitexAgent extends Thread {
@@ -57,10 +59,22 @@ public void run() {
     }
 
     private void sendStats() {
+        for (FredBoat fb : FredBoat.getShards()) {
+            if(fb.getJda().getStatus() !=  JDA.Status.CONNECTED) {
+                log.warn("Skipping posting stats because not all shards are online!");
+                return;
+            }
+        }
+
+        if (FredBoat.getShards().size() < Config.CONFIG.getNumShards()) {
+            log.warn("Skipping posting stats because not all shards initialized!");
+            return;
+        }
+
         try {
             final String response = Unirest.post("https://www.carbonitex.net/discord/data/botdata.php")
                     .field("key", key)
-                    .field("servercount", FredBoat.getAllGuilds().size())
+                    .field("servercount", FredBoat.countAllGuilds())
                     .asString().getBody();
             log.info("Successfully posted the bot data to carbonitex.com: " + response);
         } catch (Exception e) {

FredBoat/src/main/java/fredboat/agent/ShardWatchdogAgent.java

@@ -28,10 +28,12 @@
 import fredboat.Config;
 import fredboat.FredBoat;
 import fredboat.event.ShardWatchdogListener;
-import fredboat.util.DistributionEnum;
+import fredboat.util.constant.DistributionEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
 import java.util.List;
 
 public class ShardWatchdogAgent extends Thread {
@@ -69,18 +71,25 @@ public void run() {
     private void inspect() throws InterruptedException {
         List<FredBoat> shards = FredBoat.getShards();
 
-        for(FredBoat shard : shards) {
+        for (FredBoat shard : shards) {
             if (shutdown) break;
             ShardWatchdogListener listener = shard.getShardWatchdogListener();
 
             long diff = System.currentTimeMillis() - listener.getLastEventTime();
 
-            if(diff > ACCEPTABLE_SILENCE) {
+            if (diff > ACCEPTABLE_SILENCE) {
                 if (listener.getEventCount() < 100) {
                     log.warn("Did not revive shard " + shard.getShardInfo() + " because it did not receive enough events since construction!");
                 } else {
                     log.warn("Reviving shard " + shard.getShardInfo() + " after " + (diff / 1000) +
                             " seconds of no events. Last event received was " + listener.getLastEvent());
+
+                    /*try {
+                        log.info("Thread dump for shard's JDA threads at time of death: " + getShardThreadDump(shard.getShardInfo().getShardId()));
+                    } catch (Exception e) {
+                        log.error("Got exception while printing thread dump after shard death was detected");
+                    }*/
+
                     shard.revive();
                     sleep(5000);
                 }
@@ -93,10 +102,23 @@ public void shutdown() {
     }
 
     private static int getAcceptableSilenceThreshold() {
-        if(Config.CONFIG.getDistribution() == DistributionEnum.DEVELOPMENT) {
+        if (Config.CONFIG.getDistribution() == DistributionEnum.DEVELOPMENT) {
             return Integer.MAX_VALUE;
         }
 
         return Config.CONFIG.getNumShards() != 1 ? 30 * 1000 : 600 * 1000; //30 seconds or 10 minutes depending on shard count
     }
+
+    private static String getShardThreadDump(int shardId) {
+        ThreadInfo[] threadInfos = ManagementFactory.getThreadMXBean()
+                .dumpAllThreads(true,
+                        true);
+        StringBuilder dump = new StringBuilder();
+        dump.append(String.format("%n"));
+        for (ThreadInfo threadInfo : threadInfos) {
+            if (threadInfo.getThreadName().contains("[" + shardId + " / "))
+                dump.append(threadInfo);
+        }
+        return dump.toString();
+    }
 }

FredBoat/src/main/java/fredboat/api/API.java

@@ -85,8 +85,8 @@ public static void start() {
             g.put("playingPlayers", PlayerRegistry.getPlayingPlayers().size())
                     .put("totalPlayers", PlayerRegistry.getRegistry().size())
                     .put("distribution", Config.CONFIG.getDistribution())
-                    .put("guilds", FredBoat.getAllGuilds().size())
-                    .put("users", FredBoat.getAllUsersAsMap().size());
+                    .put("guilds", FredBoat.countAllGuilds())
+                    .put("users", FredBoat.countAllUniqueUsers());
 
             root.put("shards", a);
             root.put("global", g);

FredBoat/src/main/java/fredboat/audio/AbstractPlayer.java

@@ -49,7 +49,7 @@
 import fredboat.audio.queue.TrackEndMarkerHandler;
 import fredboat.audio.source.PlaylistImportSourceManager;
 import fredboat.audio.source.SpotifyPlaylistSourceManager;
-import fredboat.util.DistributionEnum;
+import fredboat.util.constant.DistributionEnum;
 import net.dv8tion.jda.core.audio.AudioSendHandler;
 import org.slf4j.LoggerFactory;
 

FredBoat/src/main/java/fredboat/command/config/GuildSettingCommand.java → FredBoat/src/main/java/fredboat/audio/DebugConnectionListener.java

@@ -23,29 +23,44 @@
  *
  */
 
-package fredboat.command.config;
+package fredboat.audio;
 
-import fredboat.commandmeta.abs.Command;
-import fredboat.commandmeta.abs.IModerationCommand;
-import net.dv8tion.jda.core.JDA;
-import net.dv8tion.jda.core.entities.Guild;
-import net.dv8tion.jda.core.entities.Member;
-import net.dv8tion.jda.core.entities.Message;
-import net.dv8tion.jda.core.entities.TextChannel;
 
-import java.util.ArrayList;
+import fredboat.FredBoat;
+import net.dv8tion.jda.core.audio.hooks.ConnectionListener;
+import net.dv8tion.jda.core.audio.hooks.ConnectionStatus;
+import net.dv8tion.jda.core.entities.User;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public abstract class GuildSettingCommand extends Command implements IModerationCommand {
+class DebugConnectionListener implements ConnectionListener {
+
+    private static final Logger log = LoggerFactory.getLogger(DebugConnectionListener.class);
+
+    private ConnectionStatus oldStatus = null;
+    private final String guildId;
+    private final FredBoat.ShardInfo shardInfo;
+
+    DebugConnectionListener(String guildId, FredBoat.ShardInfo shardInfo) {
+        this.guildId = guildId;
+        this.shardInfo = shardInfo;
+    }
+
+    @Override
+    public void onPing(long l) {
+
+    }
 
-    //TODO: Finish this
     @Override
-    public void onInvoke(JDA jda, Guild guild, TextChannel channel, Member invoker, Message message, ArrayList<String> args) {
-        super.onInvoke(jda, guild, channel, invoker, message, args);
+    public void onStatusChange(ConnectionStatus connectionStatus) {
+        log.debug(String.format("Status change for audio connection in guild %s in %s: %s => %s",
+                guildId, shardInfo, oldStatus, connectionStatus));
+
+        oldStatus = connectionStatus;
     }
 
-    //TODO: when finished, i18n the help string
     @Override
-    public String help(Guild guild) {
-        return "{0}{1}\n#TODO";
+    public void onUserSpeaking(User user, boolean b) {
+
     }
 }

FredBoat/src/main/java/fredboat/audio/GuildPlayer.java

@@ -35,6 +35,8 @@
 import fredboat.db.EntityReader;
 import fredboat.db.entity.GuildConfig;
 import fredboat.feature.I18n;
+import fredboat.perms.PermissionLevel;
+import fredboat.perms.PermsUtil;
 import fredboat.util.TextUtils;
 import net.dv8tion.jda.core.JDA;
 import net.dv8tion.jda.core.Permission;
@@ -84,6 +86,10 @@ public void joinChannel(VoiceChannel targetChannel) throws MessagingException {
         if (targetChannel == null) {
             throw new MessagingException(I18n.get(getGuild()).getString("playerUserNotInChannel"));
         }
+        if (targetChannel.equals(getChannel())) {
+            // already connected to the channel
+            return;
+        }
 
         if (!targetChannel.getGuild().getSelfMember().hasPermission(targetChannel, Permission.VOICE_CONNECT)
                 && !targetChannel.getMembers().contains(getGuild().getSelfMember())) {
@@ -98,6 +104,8 @@ public void joinChannel(VoiceChannel targetChannel) throws MessagingException {
 
         manager.openAudioConnection(targetChannel);
 
+        manager.setConnectionListener(new DebugConnectionListener(guildId, shard.getShardInfo()));
+
         log.info("Connected to voice channel " + targetChannel);
     }
 
@@ -270,7 +278,7 @@ public void setCurrentTC(TextChannel currentTC) {
     /**
      * @return currently used TextChannel or null if there is none
      */
-    public TextChannel getCurrentTC() {
+    private TextChannel getCurrentTC() {
         try {
             return shard.getJda().getTextChannelById(currentTCId);
         } catch (IllegalArgumentException e) {
@@ -280,7 +288,7 @@ public TextChannel getCurrentTC() {
 
     //Success, fail message
     public Pair<Boolean, String> canMemberSkipTracks(TextChannel textChannel, Member member, List<AudioTrackContext> list) {
-        if (member.hasPermission(textChannel, Permission.MESSAGE_MANAGE)) {
+        if (PermsUtil.checkPerms(PermissionLevel.DJ, member)) {
             return new ImmutablePair<>(true, null);
         } else {
             //We are not a mod
@@ -290,7 +298,7 @@ public Pair<Boolean, String> canMemberSkipTracks(TextChannel textChannel, Member
                 if(!atc.getMember().equals(member)) otherPeoplesTracks++;
             }
 
-            if (otherPeoplesTracks > 1) {
+            if (otherPeoplesTracks > 0) {
                 return new ImmutablePair<>(false, I18n.get(getGuild()).getString("skipDeniedTooManyTracks"));
             } else {
                 return new ImmutablePair<>(true, null);

FredBoat/src/main/java/fredboat/audio/MusicPersistenceHandler.java

@@ -34,8 +34,8 @@
 import fredboat.audio.queue.RepeatMode;
 import fredboat.audio.queue.SplitAudioTrackContext;
 import fredboat.feature.I18n;
-import fredboat.util.DistributionEnum;
-import fredboat.util.ExitCodes;
+import fredboat.util.constant.DistributionEnum;
+import fredboat.util.constant.ExitCodes;
 import net.dv8tion.jda.core.entities.Member;
 import net.dv8tion.jda.core.entities.TextChannel;
 import net.dv8tion.jda.core.entities.VoiceChannel;