Merge pull request #57 from disney/develop

Develop to master for 0.9.10

.gitignore

@@ -21,3 +21,5 @@ bin/
 
 # Automation
 .espnautomation/
+.DS_Store
+quanta-loader/quanta-loader

Docker/DeployKinesisConsumerDockerfile

@@ -27,4 +27,6 @@ RUN chmod 755 /usr/bin/entrypoint.sh
 # Port mapping of MySQL Proxy server. Default("127.0.0.1:4000")
 EXPOSE 2112
 
-ENTRYPOINT ["/usr/bin/entrypoint.sh"]
+STOPSIGNAL SIGINT
+
+CMD ["/usr/bin/entrypoint.sh"]

Docker/kinesis_entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+
 BOOL_FLAGS=""
 if [ -n "$INITIAL_POSITION" ]
 then
@@ -39,13 +40,16 @@ then
     	BOOL_FLAGS=${BOOL_FLAGS}" --assume-role-arn-region=${ASSUME_ROLE_ARN_REGION}"
 	fi
 fi
-if [ -n "$SESSION_POOL_SIZE" ]
-then
-    BOOL_FLAGS=${BOOL_FLAGS}" --session-pool-size=${SESSION_POOL_SIZE}"
-fi
 if [ -n "$PRESELECTOR" ]
 then
     BOOL_FLAGS=${BOOL_FLAGS}" --preselector="${PRESELECTOR}""
 fi
-exec /usr/bin/quanta-kinesis-consumer ${STREAM} ${INDEX} ${REGION} ${BOOL_FLAGS}
-
+if [ -n "$SCAN_INTERVAL" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --scan-interval="${SCAN_INTERVAL}""
+fi
+if [ -n "$COMMIT_INTERVAL" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --commit-interval="${COMMIT_INTERVAL}""
+fi
+exec /usr/bin/quanta-kinesis-consumer ${STREAM} ${INDEX} ${SHARD_KEY} ${REGION} ${BOOL_FLAGS}

Docker/loader_entrypoint.sh

@@ -1,10 +1,27 @@
 #!/bin/sh
 BOOL_FLAGS=""
-DATE_FILTER_FLAG=""
-
-if [ -n "$DATE_FILTER" ] 
+if [ -n "$IGNORE_SOURCE_PATH" ]
 then
-    DATE_FILTER_FLAG="--date-filter $DATE_FILTER"
+    BOOL_FLAGS=${BOOL_FLAGS}" --ignore-source-path"
 fi
-
-/usr/bin/quanta-loader ${BOOL_FLAGS} ${DATE_FILTER_FLAG} --buf-size ${BUF_SIZE} ${BUCKET_PATH} ${INDEX}
+if [ -n "$NERD_CAPITALIZATION" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --nerd-capitalization"
+fi
+if [ -n "$ROLE_ARN" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --role-arn=${ROLE_ARN}"
+fi
+if [ -n "$AWS_REGION" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --aws-region=${AWS_REGION}"
+fi
+if [ -n "$KMS_KEY_ID" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --kms-key-id=${KMS_KEY_ID}"
+fi
+if [ -n "$ACL" ]
+then
+    BOOL_FLAGS=${BOOL_FLAGS}" --acl=${ACL}"
+fi
+/usr/bin/quanta-loader ${BUCKET_PATH} ${INDEX} --buf-size ${BUF_SIZE} ${BOOL_FLAGS}

Makefile

@@ -58,7 +58,6 @@ vet:
 lint:
 	go get golang.org/x/lint/golint
 	golint -set_exit_status ${GOLIST}
-	#gas ${PKG}
 
 format:
 	go fmt ${PKG}
@@ -94,6 +93,7 @@ admin:
 
 loader:
 	go build -o ${BIN_DIR}/${BIN_LOADER} ${LDFLAGS} ${PKG_LOADER}
+	docker build -t containerregistry.disney.com/digital/quanta-loader -f Docker/DeployLoaderDockerfile .
 
 producer:
 	go build -o ${BIN_DIR}/${BIN_PRODUCER} ${LDFLAGS} ${PKG_PRODUCER}

core/builtins.go

@@ -103,6 +103,7 @@ func (m BoolDirectMapper) MapValue(attr *Attribute, val interface{},
 			result = uint64(1)
 		}
 	default:
+		err = fmt.Errorf("%v: No handling for type '%T'", val, val)
 		return
 	}
 	if c != nil {
@@ -221,6 +222,8 @@ func (m FloatScaleBSIMapper) MapValue(attr *Attribute, val interface{},
 	switch val.(type) {
 	case float64:
 		floatVal = val.(float64)
+	case float32:
+		floatVal = float64(val.(float32))
 	case uint64:
 		floatVal = float64(val.(uint64))
 	case int64:
@@ -438,6 +441,10 @@ func (m SysMillisBSIMapper) MapValue(attr *Attribute, val interface{},
 	switch val.(type) {
 	case string:
 		strVal := val.(string)
+		if strVal == "" || strVal == "NULL" {
+			result = uint64(0)
+			return
+		}
 		loc, _ := time.LoadLocation("Local")
 		var t time.Time
 		t, err = dateparse.ParseIn(strVal, loc)
@@ -482,6 +489,10 @@ func (m SysMicroBSIMapper) MapValue(attr *Attribute, val interface{},
 	switch val.(type) {
 	case string:
 		strVal := val.(string)
+		if strVal == "" || strVal == "NULL" {
+			result = uint64(0)
+			return
+		}
 		loc, _ := time.LoadLocation("Local")
 		var t time.Time
 		t, err = dateparse.ParseIn(strVal, loc)
@@ -526,6 +537,10 @@ func (m SysSecBSIMapper) MapValue(attr *Attribute, val interface{},
 	switch val.(type) {
 	case string:
 		strVal := val.(string)
+		if strVal == "" || strVal == "NULL" {
+			result = uint64(0)
+			return
+		}
 		loc, _ := time.LoadLocation("Local")
 		var t time.Time
 		t, err = dateparse.ParseIn(strVal, loc)

core/projector.go

@@ -154,6 +154,7 @@ func NewProjection(s *Session, foundSets map[string]*roaring64.Bitmap, joinNames
 
 	p.resultIterator = driverSet.ManyIterator()
 
+//u.Warnf("PROJ FIELDS = %#v", p.projFieldMap)
 	return p, nil
 }