Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

started refactoring towards java objects, but didn't finish #2

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

started refactoring towards java objects, but didn't finish #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2d3dba8
started refactoring towards java objects, but didn't finish
rfecher May 18, 2016
dc02356
example of SPI
rfecher May 19, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...ETA-INF/services/mil.nga.giat.geowave.core.store.data.field.FieldSerializationProviderSpi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
com.example.ingest.vector.ScalaDoubleSerializationProvider
58 changes: 29 additions & 29 deletions src/main/scala/GDeltLine.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,34 +82,34 @@ object GDeltLine {
}
}

class GDeltLine (val globalEventId: Int, val day: Int,
val monthYear: Int, val year: Int,
val fractionDate: Double, val actor1Code: String,
val actor1Name: String, val actor1CountryCode: String,
val actor1KnownGroupCode: String, val actor1EthnicCode: String,
val actor1Religion1Code: String, val actor1Religion2Code: String,
val actor1Type1Code: String, val actor1Type2Code: String,
val actor1Type3Code: String, val actor2Code: String,
val actor2Name: String, val actor2CountryCode: String,
val actor2KnownGroupCode: String, val actor2EthnicCode: String,
val actor2Religion1Code: String, val actor2Religion2Code: String,
val actor2Type1Code: String, val actor2Type2Code: String,
val actor2Type3Code: String, val isRootEvent: Int,
val eventCode: String, val eventBaseCode: String,
val eventRootCode: String, val quadClass: Int,
val goldsteinScale: Double, val numMentions: Int,
val numSources: Int, val numArticles: Int,
val avgTone: Double, val actor1Geo_Type: Int,
val actor1Geo_Fullname: String, val actor1Geo_CountryCode: String,
val actor1Geo_ADM1Code: String, val actor1Geo_Lat: Double,
val actor1Geo_Long: Double, val actor1Geo_FeatureID: Int,
val actor2Geo_Type: Int, val actor2Geo_Fullname: String,
val actor2Geo_CountryCode: String,val actor2Geo_ADM1Code: String,
val actor2Geo_Lat: Double, val actor2Geo_Long: Double,
val actor2Geo_FeatureID: Int, val actionGeo_Type: Int,
val actionGeo_Fullname: String, val actionGeo_CountryCode: String,
val actionGeo_ADM1Code: String, val actionGeo_Lat: Double,
val actionGeo_Long: Double, val actionGeo_FeatureID: Int,
val dateAdded: Int, val sourceURL: String)
class GDeltLine (val globalEventId: Integer, val day: Integer,
val monthYear: Integer, val year: Integer,
val fractionDate: java.lang.Double, val actor1Code: String,
val actor1Name: String, val actor1CountryCode: String,
val actor1KnownGroupCode: String, val actor1EthnicCode: String,
val actor1Religion1Code: String, val actor1Religion2Code: String,
val actor1Type1Code: String, val actor1Type2Code: String,
val actor1Type3Code: String, val actor2Code: String,
val actor2Name: String, val actor2CountryCode: String,
val actor2KnownGroupCode: String, val actor2EthnicCode: String,
val actor2Religion1Code: String, val actor2Religion2Code: String,
val actor2Type1Code: String, val actor2Type2Code: String,
val actor2Type3Code: String, val isRootEvent: Integer,
val eventCode: String, val eventBaseCode: String,
val eventRootCode: String, val quadClass: Integer,
val goldsteinScale: java.lang.Double,val numMentions: Integer,
val numSources: Integer, val numArticles: Integer,
val avgTone: java.lang.Double, val actor1Geo_Type: Integer,
val actor1Geo_Fullname: String, val actor1Geo_CountryCode: String,
val actor1Geo_ADM1Code: String, val actor1Geo_Lat: java.lang.Double,
val actor1Geo_Long: java.lang.Double,val actor1Geo_FeatureID: Integer,
val actor2Geo_Type: Integer, val actor2Geo_Fullname: String,
val actor2Geo_CountryCode: String, val actor2Geo_ADM1Code: String,
val actor2Geo_Lat: java.lang.Double, val actor2Geo_Long: java.lang.Double,
val actor2Geo_FeatureID: Integer, val actionGeo_Type: Integer,
val actionGeo_Fullname: String, val actionGeo_CountryCode: String,
val actionGeo_ADM1Code: String, val actionGeo_Lat: java.lang.Double,
val actionGeo_Long: java.lang.Double,val actionGeo_FeatureID: Integer,
val dateAdded: Integer, val sourceURL: String)


48 changes: 25 additions & 23 deletions src/main/scala/GdeltIngest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.example.ingest.vector

import java.lang

import org.apache.log4j.Logger

import mil.nga.giat.geowave.adapter.vector._
Expand Down Expand Up @@ -88,15 +90,15 @@ object GdeltIngest {
// having to handle geometries.

// GlobalEventId
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(0)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(0)))
// Day
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(1)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(1)))
// MonthYear
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(2)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(2)))
// Year
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(3)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(3)))
// FractionDate
builder.add(ab.binding(classOf[Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(4)))
builder.add(ab.binding(classOf[lang.Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(4)))
// Actor1Code
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(5)))
// Actor1Name
Expand Down Expand Up @@ -138,25 +140,25 @@ object GdeltIngest {
// Actor2Type3Code
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(24)))
// IsRootEvent
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(25)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(25)))
// EventCode
builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(26)))
// EventBaseCode
builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(27)))
// EventRootCode
builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(28)))
// QuadClass
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(29)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(29)))
// GoldsteinScale
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(30)))
// NumMentions
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(31)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(31)))
// NumSources
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(32)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(32)))
// NumArticles
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(33)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(33)))
// AvgTone
builder.add(ab.binding(classOf[Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(34)))
builder.add(ab.binding(classOf[lang.Double]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(34)))
// Actor1Geo_Type
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(35)))
// Actor1Geo_Fullname
Expand All @@ -166,41 +168,41 @@ object GdeltIngest {
// Actor1Geo_ADM1Code
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(38)))
// Actor1Geo_Lat
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(39)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(39)))
// Actor1Geo_Long
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(40)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(40)))
// Actor1Geo_FeatureID
builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(41)))
builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(41)))
// Actor2Geo_Type
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(42)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(42)))
// Actor2Geo_Fullname
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(43)))
// Actor2Geo_CountryCode
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(44)))
// Actor2Geo_ADM1Code
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(45)))
// Actor2Geo_Lat
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(46)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(46)))
// Actor2Geo_Long
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(47)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(47)))
// Actor2Geo_FeatureID
builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(48)))
builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(48)))
// ActionGeo_Type
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(49)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(49)))
// ActionGeo_Fullname
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(50)))
// ActionGeo_CountryCode
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(51)))
// ActionGeo_ADM1Code
builder.add(ab.binding(classOf[String]).buildDescriptor(GDeltLine.gdeltCodeNames(52)))
// ActionGeo_Lat
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(53)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(53)))
// ActionGeo_Long
builder.add(ab.binding(classOf[Double]).buildDescriptor(GDeltLine.gdeltCodeNames(54)))
builder.add(ab.binding(classOf[lang.Double]).buildDescriptor(GDeltLine.gdeltCodeNames(54)))
// ActionGeo_FeatureID
builder.add(ab.binding(classOf[Int]).buildDescriptor(GDeltLine.gdeltCodeNames(55)))
builder.add(ab.binding(classOf[Integer]).buildDescriptor(GDeltLine.gdeltCodeNames(55)))
// DateAdded
builder.add(ab.binding(classOf[Int]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(56)))
builder.add(ab.binding(classOf[Integer]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(56)))
// SourceURL
builder.add(ab.binding(classOf[String]).nillable(false).buildDescriptor(GDeltLine.gdeltCodeNames(57)))

Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/Main.scala → src/main/scala/GdeltIngestMain.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ object GdeltIngestMain {
List(scala.io.Source.fromFile(filename))
}).map(_.getLines)

val maybeBAO = Try(GdeltIngest.getAccumuloOperationsInstance("leader","instance","root","password","gwGDELT"))
val maybeBAO = Try(GdeltIngest.getAccumuloOperationsInstance("localhost:2181","geowave","root","password","gwGDELT"))
if (maybeBAO.isFailure) {
println("Could not create Accumulo instance")
println(maybeBAO)
Expand Down
19 changes: 19 additions & 0 deletions src/main/scala/ScalaDoubleSerializationProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.example.ingest.vector

import java.lang

import mil.nga.giat.geowave.core.index.ByteArrayId
import mil.nga.giat.geowave.core.store.data.field.base.DoubleSerializationProvider
import mil.nga.giat.geowave.core.store.data.field.{FieldWriter, FieldReader, FieldSerializationProviderSpi}

class ScalaDoubleSerializationProvider extends FieldSerializationProviderSpi[Double]{
val delegate = new DoubleSerializationProvider

// override def getFieldReader: FieldReader[lang.Double] = delegate.getFieldReader

// override def getFieldWriter: FieldWriter[AnyRef, lang.Double] = delegate.getFieldWriter
override def getFieldWriter: FieldWriter[AnyRef, Double] = ???


override def getFieldReader: FieldReader[Double] = ???
}