diff --git a/src/main/resources/META-INF/services/mil.nga.giat.geowave.core.store.data.field.FieldSerializationProviderSpi b/src/main/resources/META-INF/services/mil.nga.giat.geowave.core.store.data.field.FieldSerializationProviderSpi new file mode 100644 index 0000000..116aca5 --- /dev/null +++ b/src/main/resources/META-INF/services/mil.nga.giat.geowave.core.store.data.field.FieldSerializationProviderSpi @@ -0,0 +1 @@ +com.example.ingest.vector.ScalaDoubleSerializationProvider \ No newline at end of file diff --git a/src/main/scala/GDeltLine.scala b/src/main/scala/GDeltLine.scala index f55daa2..14e2955 100644 --- a/src/main/scala/GDeltLine.scala +++ b/src/main/scala/GDeltLine.scala @@ -82,34 +82,34 @@ object GDeltLine { } } -class GDeltLine (val globalEventId: Int, val day: Int, - val monthYear: Int, val year: Int, - val fractionDate: Double, val actor1Code: String, - val actor1Name: String, val actor1CountryCode: String, - val actor1KnownGroupCode: String, val actor1EthnicCode: String, - val actor1Religion1Code: String, val actor1Religion2Code: String, - val actor1Type1Code: String, val actor1Type2Code: String, - val actor1Type3Code: String, val actor2Code: String, - val actor2Name: String, val actor2CountryCode: String, - val actor2KnownGroupCode: String, val actor2EthnicCode: String, - val actor2Religion1Code: String, val actor2Religion2Code: String, - val actor2Type1Code: String, val actor2Type2Code: String, - val actor2Type3Code: String, val isRootEvent: Int, - val eventCode: String, val eventBaseCode: String, - val eventRootCode: String, val quadClass: Int, - val goldsteinScale: Double, val numMentions: Int, - val numSources: Int, val numArticles: Int, - val avgTone: Double, val actor1Geo_Type: Int, - val actor1Geo_Fullname: String, val actor1Geo_CountryCode: String, - val actor1Geo_ADM1Code: String, val actor1Geo_Lat: Double, - val actor1Geo_Long: Double, val actor1Geo_FeatureID: Int, - val actor2Geo_Type: Int, val actor2Geo_Fullname: String, - val actor2Geo_CountryCode: String,val actor2Geo_ADM1Code: String, - val actor2Geo_Lat: Double, val actor2Geo_Long: Double, - val actor2Geo_FeatureID: Int, val actionGeo_Type: Int, - val actionGeo_Fullname: String, val actionGeo_CountryCode: String, - val actionGeo_ADM1Code: String, val actionGeo_Lat: Double, - val actionGeo_Long: Double, val actionGeo_FeatureID: Int, - val dateAdded: Int, val sourceURL: String) +class GDeltLine (val globalEventId: Integer, val day: Integer, + val monthYear: Integer, val year: Integer, + val fractionDate: java.lang.Double, val actor1Code: String, + val actor1Name: String, val actor1CountryCode: String, + val actor1KnownGroupCode: String, val actor1EthnicCode: String, + val actor1Religion1Code: String, val actor1Religion2Code: String, + val actor1Type1Code: String, val actor1Type2Code: String, + val actor1Type3Code: String, val actor2Code: String, + val actor2Name: String, val actor2CountryCode: String, + val actor2KnownGroupCode: String, val actor2EthnicCode: String, + val actor2Religion1Code: String, val actor2Religion2Code: String, + val actor2Type1Code: String, val actor2Type2Code: String, + val actor2Type3Code: String, val isRootEvent: Integer, + val eventCode: String, val eventBaseCode: String, + val eventRootCode: String, val quadClass: Integer, + val goldsteinScale: java.lang.Double,val numMentions: Integer, + val numSources: Integer, val numArticles: Integer, + val avgTone: java.lang.Double, val actor1Geo_Type: Integer, + val actor1Geo_Fullname: String, val actor1Geo_CountryCode: String, + val actor1Geo_ADM1Code: String, val actor1Geo_Lat: java.lang.Double, + val actor1Geo_Long: java.lang.Double,val actor1Geo_FeatureID: Integer, + val actor2Geo_Type: Integer, val actor2Geo_Fullname: String, + val actor2Geo_CountryCode: String, val actor2Geo_ADM1Code: String, + val actor2Geo_Lat: java.lang.Double, val actor2Geo_Long: java.lang.Double, + val actor2Geo_FeatureID: Integer, val actionGeo_Type: Integer, + val actionGeo_Fullname: String, val actionGeo_CountryCode: String, + val actionGeo_ADM1Code: String, val actionGeo_Lat: java.lang.Double, + val actionGeo_Long: java.lang.Double,val actionGeo_FeatureID: Integer, + val dateAdded: Integer, val sourceURL: String) diff --git a/src/main/scala/GdeltIngest.scala b/src/main/scala/GdeltIngest.scala index 4801e43..98121fa 100644 --- a/src/main/scala/GdeltIngest.scala +++ b/src/main/scala/GdeltIngest.scala @@ -1,5 +1,7 @@ package com.example.ingest.vector +import java.lang + import org.apache.log4j.Logger import mil.nga.giat.geowave.adapter.vector._ @@ -88,15 +90,15 @@ object GdeltIngest { // having to handle geometries. // GlobalEventId - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(0))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(0))) // Day - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(1))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(1))) // MonthYear - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(2))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(2))) // Year - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(3))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(3))) // FractionDate - builder.add(ab.binding(classOf[Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(4))) + builder.add(ab.binding(classOf[lang.Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(4))) // Actor1Code builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(5))) // Actor1Name @@ -138,7 +140,7 @@ object GdeltIngest { // Actor2Type3Code builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(24))) // IsRootEvent - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(25))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(25))) // EventCode builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(26))) // EventBaseCode @@ -146,17 +148,17 @@ object GdeltIngest { // EventRootCode builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(28))) // QuadClass - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(29))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(29))) // GoldsteinScale builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(30))) // NumMentions - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(31))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(31))) // NumSources - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(32))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(32))) // NumArticles - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(33))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(33))) // AvgTone - builder.add(ab.binding(classOf[Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(34))) + builder.add(ab.binding(classOf[lang.Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(34))) // Actor1Geo_Type builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(35))) // Actor1Geo_Fullname @@ -166,13 +168,13 @@ object GdeltIngest { // Actor1Geo_ADM1Code builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(38))) // Actor1Geo_Lat - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(39))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(39))) // Actor1Geo_Long - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(40))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(40))) // Actor1Geo_FeatureID - builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(41))) + builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(41))) // Actor2Geo_Type - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(42))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(42))) // Actor2Geo_Fullname builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(43))) // Actor2Geo_CountryCode @@ -180,13 +182,13 @@ object GdeltIngest { // Actor2Geo_ADM1Code builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(45))) // Actor2Geo_Lat - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(46))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(46))) // Actor2Geo_Long - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(47))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(47))) // Actor2Geo_FeatureID - builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(48))) + builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(48))) // ActionGeo_Type - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(49))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(49))) // ActionGeo_Fullname builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(50))) // ActionGeo_CountryCode @@ -194,13 +196,13 @@ object GdeltIngest { // ActionGeo_ADM1Code builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(52))) // ActionGeo_Lat - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(53))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(53))) // ActionGeo_Long - builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(54))) + builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(54))) // ActionGeo_FeatureID - builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(55))) + builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(55))) // DateAdded - builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(56))) + builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(56))) // SourceURL builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(57))) diff --git a/src/main/scala/Main.scala b/src/main/scala/GdeltIngestMain.scala similarity index 98% rename from src/main/scala/Main.scala rename to src/main/scala/GdeltIngestMain.scala index 244f34d..0405838 100644 --- a/src/main/scala/Main.scala +++ b/src/main/scala/GdeltIngestMain.scala @@ -121,7 +121,7 @@ object GdeltIngestMain { List(scala.io.Source.fromFile(filename)) }).map(_.getLines) - val maybeBAO = Try(GdeltIngest.getAccumuloOperationsInstance("leader","instance","root","password","gwGDELT")) + val maybeBAO = Try(GdeltIngest.getAccumuloOperationsInstance("localhost:2181","geowave","root","password","gwGDELT")) if (maybeBAO.isFailure) { println("Could not create Accumulo instance") println(maybeBAO) diff --git a/src/main/scala/ScalaDoubleSerializationProvider.scala b/src/main/scala/ScalaDoubleSerializationProvider.scala new file mode 100644 index 0000000..325f461 --- /dev/null +++ b/src/main/scala/ScalaDoubleSerializationProvider.scala @@ -0,0 +1,19 @@ +package com.example.ingest.vector + +import java.lang + +import mil.nga.giat.geowave.core.index.ByteArrayId +import mil.nga.giat.geowave.core.store.data.field.base.DoubleSerializationProvider +import mil.nga.giat.geowave.core.store.data.field.{FieldWriter, FieldReader, FieldSerializationProviderSpi} + +class ScalaDoubleSerializationProvider extends FieldSerializationProviderSpi[Double]{ + val delegate = new DoubleSerializationProvider + +// override def getFieldReader: FieldReader[lang.Double] = delegate.getFieldReader + +// override def getFieldWriter: FieldWriter[AnyRef, lang.Double] = delegate.getFieldWriter + override def getFieldWriter: FieldWriter[AnyRef, Double] = ??? + + + override def getFieldReader: FieldReader[Double] = ??? +}