Merge pull request #333 from RedisLabs/acl-auth

issue #328: add user/password authentication (Redis 6 ACL feature)
RedisLabs · Jan 14, 2022 · 7d2b723 · 7d2b723
2 parents 1b72192 + ddb174a
commit 7d2b723
Show file tree

Hide file tree

Showing 23 changed files with 372 additions and 95 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,5 +1,25 @@
 version: 2.1
+
 commands:
+
+  abort_for_docs:
+    steps:
+      - run:
+          name: Avoid tests for docs
+          command: |
+            if [[ $CIRCLE_BRANCH == *docs ]]; then
+              echo "Identifies as documents PR, no testing required"
+              circleci step halt
+            fi
+  abort_for_noci:
+    steps:
+      - run:
+          name: Ignore CI for specific branches
+          command: |
+            if [[ $CIRCLE_BRANCH == *noci ]]; then
+              echo "Identifies as actively ignoring CI, no testing required."
+              circleci step halt
+            fi
   early_return_for_forked_pull_requests:
     description: >-
       If this build is from a fork, stop executing the current job and return success.
@@ -12,80 +32,140 @@ commands:
               echo "Nothing to do for forked PRs, so marking this step successful"
               circleci step halt
             fi
+jobs:
 
-executors:
-  linux-8-jdk:
+  build:
     docker:
       - image: circleci/openjdk:8-jdk
 
-jobs:
-  build:
-    parameters:
-      os:
-        type: executor
-
-    executor: << parameters.os >>
-
-    working_directory: ~/repo
-
     environment:
       JVM_OPTS: -Xmx3200m
       TERM: dumb
 
     steps:
+      - abort_for_docs
+      - abort_for_noci
       - checkout
 
       # Download and cache dependencies
       - restore_cache:
           keys:
             - spark-redis-{{ checksum "pom.xml" }}
 
-      - run: mvn dependency:go-offline 
+      - run: mvn dependency:go-offline
 
+      - run:
+          name: install Redis
+          command: |
+            sudo apt-get -y update
+            sudo apt-get install -y libssl-dev
+            wget http://download.redis.io/releases/redis-6.0.10.tar.gz
+            tar -xzvf redis-6.0.10.tar.gz
+            make -C redis-6.0.10 -j`nproc` BUILD_TLS=yes
+            export PATH=$PWD/redis-6.0.10/src:$PATH
+      - run:
+          name: Run Test
+          command: |
+            export PATH=$PWD/redis-6.0.10/src:$PATH
+            make test
+      - run:
+          name: switch scala to 2.12
+          command: |
+            sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile)
+            ps aux | grep redis
+            ./dev/change-scala-version.sh 2.12 # switch to scala 2.12
+      - run:
+          name: Run Test with scala 2.12
+          command: |
+            export PATH=$PWD/redis-6.0.10/src:$PATH
+            make test
       - save_cache:
           paths:
             - ~/.m2
           key: spark-redis-{{ checksum "pom.xml" }}
 
-      - run: 
+      - early_return_for_forked_pull_requests
+
+      - run: bash <(curl -s https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t ${CODECOV_TOKEN}
+
+  build-and-publish:
+    docker:
+      - image: circleci/openjdk:8-jdk
+
+    environment:
+      JVM_OPTS: -Xmx3200m
+      TERM: dumb
+
+    steps:
+      - abort_for_docs
+      - abort_for_noci
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - spark-redis-{{ checksum "pom.xml" }}
+
+      - run: mvn dependency:go-offline
+
+      - run:
           name: install Redis
           command: |
             sudo apt-get -y update
-            sudo apt-get install -y gcc-8 g++-8 libssl-dev
+            sudo apt-get install -y libssl-dev
             wget http://download.redis.io/releases/redis-6.0.10.tar.gz
             tar -xzvf redis-6.0.10.tar.gz
-            make -C redis-6.0.10 -j4 BUILD_TLS=yes
+            make -C redis-6.0.10 -j`nproc` BUILD_TLS=yes
             export PATH=$PWD/redis-6.0.10/src:$PATH
-
       - run:
           name: Run Test
           command: |
             export PATH=$PWD/redis-6.0.10/src:$PATH
             make test
-      
       - run:
-          name: switch scala to 2.12 
+          name: switch scala to 2.12
           command: |
             sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile)
             ps aux | grep redis
             ./dev/change-scala-version.sh 2.12 # switch to scala 2.12
-      
-      - run: 
-          name: Run Test with scala 2.12     
+      - run:
+          name: Run Test with scala 2.12
           command: |
             export PATH=$PWD/redis-6.0.10/src:$PATH
             make test
-      
+      - save_cache:
+          paths:
+            - ~/.m2
+          key: spark-redis-{{ checksum "pom.xml" }}
+
       - early_return_for_forked_pull_requests
 
-      - run: bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} 
-     
+      - run: bash <(curl -s https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t ${CODECOV_TOKEN}
+
       - run: mvn -s .circleci.settings.xml -DskipTests deploy
 
+on-master: &on-master
+  filters:
+    branches:
+      only: master
+    tags:
+      ignore: /.*/
+
+not-on-master: &not-on-master
+  filters:
+    branches:
+      ignore: master
+    tags:
+      ignore: /.*/
+
 workflows:
-  all-jdks:
+  commit:
     jobs:
       - build:
-          matrix:
-            parameters:
-              os: [linux-8-jdk]
+          <<: *not-on-master
+          context:
+            - common
+      - build-and-publish:
+          <<: *on-master
+          context:
+            - common
diff --git a/Makefile b/Makefile
@@ -1,3 +1,6 @@
+# user
+USER_ACL = user alice on >p1pp0 ~* +@all
+
 # STANDALONE REDIS NODE
 define REDIS_STANDALONE_NODE_CONF
 daemonize yes
@@ -7,6 +10,7 @@ logfile /tmp/redis_standalone_node_for_spark-redis.log
 save ""
 appendonly no
 requirepass passwd
+$(USER_ACL)
 endef
 
 # STANDALONE REDIS NODE WITH SSL
@@ -18,6 +22,7 @@ logfile /tmp/redis_standalone_node_ssl_for_spark-redis.log
 save ""
 appendonly no
 requirepass passwd
+$(USER_ACL)
 tls-auth-clients no
 tls-port 6380
 tls-cert-file ./src/test/resources/tls/redis.crt
@@ -30,6 +35,7 @@ endef
 define REDIS_CLUSTER_NODE1_CONF
 daemonize yes
 port 7379
+$(USER_ACL)
 pidfile /tmp/redis_cluster_node1_for_spark-redis.pid
 logfile /tmp/redis_cluster_node1_for_spark-redis.log
 save ""
@@ -41,6 +47,7 @@ endef
 define REDIS_CLUSTER_NODE2_CONF
 daemonize yes
 port 7380
+$(USER_ACL)
 pidfile /tmp/redis_cluster_node2_for_spark-redis.pid
 logfile /tmp/redis_cluster_node2_for_spark-redis.log
 save ""
@@ -52,6 +59,7 @@ endef
 define REDIS_CLUSTER_NODE3_CONF
 daemonize yes
 port 7381
+$(USER_ACL)
 pidfile /tmp/redis_cluster_node3_for_spark-redis.pid
 logfile /tmp/redis_cluster_node3_for_spark-redis.log
 save ""

diff --git a/doc/configuration.md b/doc/configuration.md
@@ -5,6 +5,7 @@ The supported configuration parameters are:
 * `spark.redis.host` - host or IP of the initial node we connect to. The connector will read the cluster
 topology from the initial node, so there is no need to provide the rest of the cluster nodes.
 * `spark.redis.port` - the initial node's TCP redis port.
+* `spark.redis.user` - the initial node's AUTH user
 * `spark.redis.auth` - the initial node's AUTH password
 * `spark.redis.db` - optional DB number. Avoid using this, especially in cluster mode.
 * `spark.redis.timeout` - connection timeout in ms, 2000 ms by default

diff --git a/doc/dataframe.md b/doc/dataframe.md
@@ -330,22 +330,23 @@ root
 
 ## DataFrame options
 
-| Name                   | Description                                                                               | Type                  | Default |
-| -----------------------| ------------------------------------------------------------------------------------------| --------------------- | ------- |
-| model                  | defines the Redis model used to persist DataFrame, see [Persistence model](#persistence-model)| `enum [binary, hash]` | `hash`  |
-| filter.keys.by.type    | make sure the underlying data structures match persistence model                          | `Boolean`             | `false` |
-| partitions.number      | number of partitions (applies only when reading DataFrame)                                | `Int`                 | `3`     |
+| Name                   | Description                                                                                                                                                 | Type                  | Default |
+|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------------- | ------- |
+| model                  | defines the Redis model used to persist DataFrame, see [Persistence model](#persistence-model)                                                              | `enum [binary, hash]` | `hash`  |
+| filter.keys.by.type    | make sure the underlying data structures match persistence model                                                                                            | `Boolean`             | `false` |
+| partitions.number      | number of partitions (applies only when reading DataFrame)                                                                                                  | `Int`                 | `3`     |
 | key.column             | when writing - specifies unique column used as a Redis key, by default a key is auto-generated <br/> when reading - specifies column name to store hash key | `String`              | -       |
-| ttl                    | data time to live in `seconds`. Data doesn't expire if `ttl` is less than `1`             | `Int`                 | `0`     |
-| infer.schema           | infer schema from random row, all columns will have `String` type                         | `Boolean`             | `false` |
-| max.pipeline.size      | maximum number of commands per pipeline (used to batch commands)                          | `Int`                 | 100     |
-| scan.count             | count option of SCAN command (used to iterate over keys)                                  | `Int`                 | 100     |
-| iterator.grouping.size | the number of items to be grouped when iterating over underlying RDD partition            | `Int`                 | 1000    |
-| host                   | overrides `spark.redis.host` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                        | `String`              | `localhost`    |
-| port                   | overrides `spark.redis.port` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                        | `Int`                 | `6379`         |
-| auth                   | overrides `spark.redis.auth` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                        | `String`              | -              |
-| dbNum                  | overrides `spark.redis.db` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                       | `Int`                 | `0`            |
-| timeout                | overrides `spark.redis.timeout` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                     | `Int`                 | `2000`         |
+| ttl                    | data time to live in `seconds`. Data doesn't expire if `ttl` is less than `1`                                                                               | `Int`                 | `0`     |
+| infer.schema           | infer schema from random row, all columns will have `String` type                                                                                           | `Boolean`             | `false` |
+| max.pipeline.size      | maximum number of commands per pipeline (used to batch commands)                                                                                            | `Int`                 | 100     |
+| scan.count             | count option of SCAN command (used to iterate over keys)                                                                                                    | `Int`                 | 100     |
+| iterator.grouping.size | the number of items to be grouped when iterating over underlying RDD partition                                                                              | `Int`                 | 1000    |
+| host                   | overrides `spark.redis.host` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                                | `String`              | `localhost`    |
+| port                   | overrides `spark.redis.port` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                                | `Int`                 | `6379`         |
+| user                   | overrides `spark.redis.user` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                                | `String`              | -              |
+| auth                   | overrides `spark.redis.auth` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                                | `String`              | -              |
+| dbNum                  | overrides `spark.redis.db` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                                  | `Int`                 | `0`            |
+| timeout                | overrides `spark.redis.timeout` configured in SparkSession (if set, any other connection setting from SparkSession is ignored )                             | `Int`                 | `2000`         |
 
 
 ## Known limitations

diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.redislabs</groupId>
 	<artifactId>spark-redis_2.11</artifactId>
-	<version>2.6.0-SNAPSHOT</version>
+	<version>2.7.0-SNAPSHOT</version>
 	<name>Spark-Redis</name>
 	<description>A Spark library for Redis</description>
 	<url>http://github.com/RedisLabs/spark-redis</url>
@@ -49,7 +49,7 @@
 		<java.version>1.8</java.version>
 		<scala.major.version>2.11</scala.major.version>
 		<scala.complete.version>${scala.major.version}.12</scala.complete.version>
-		<jedis.version>3.2.0</jedis.version>
+		<jedis.version>3.8.0</jedis.version>
 		<spark.version>2.4.1</spark.version>
 		<plugins.scalatest.version>1.0</plugins.scalatest.version>
 	</properties>
@@ -271,11 +271,6 @@
 		</plugins>
 	</build>
 	<dependencies>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-pool2</artifactId>
-            <version>2.0</version>
-        </dependency>
         <dependency>
             <groupId>redis.clients</groupId>
             <artifactId>jedis</artifactId>

diff --git a/src/main/scala/com/redislabs/provider/redis/ConnectionPool.scala b/src/main/scala/com/redislabs/provider/redis/ConnectionPool.scala
@@ -1,10 +1,10 @@
 package com.redislabs.provider.redis
 
-import redis.clients.jedis.{JedisPoolConfig, Jedis, JedisPool}
 import redis.clients.jedis.exceptions.JedisConnectionException
+import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
 
+import java.time.Duration
 import java.util.concurrent.ConcurrentHashMap
-
 import scala.collection.JavaConversions._
 
 
@@ -21,11 +21,11 @@ object ConnectionPool {
         poolConfig.setTestOnBorrow(false)
         poolConfig.setTestOnReturn(false)
         poolConfig.setTestWhileIdle(false)
-        poolConfig.setMinEvictableIdleTimeMillis(60000)
-        poolConfig.setTimeBetweenEvictionRunsMillis(30000)
+        poolConfig.setSoftMinEvictableIdleTime(Duration.ofMinutes(1))
+        poolConfig.setTimeBetweenEvictionRuns(Duration.ofSeconds(30))
         poolConfig.setNumTestsPerEvictionRun(-1)
 
-        new JedisPool(poolConfig, re.host, re.port, re.timeout, re.auth, re.dbNum, re.ssl)
+        new JedisPool(poolConfig, re.host, re.port, re.timeout, re.user, re.auth, re.dbNum, re.ssl)
       }
     )
     var sleepTime: Int = 4