Merge pull request #14 from v3io/restore

emd ignore attrs move to conf
v3io · Jul 18, 2017 · 9c4105b · 9c4105b
2 parents ec51e49 + 5c63c7d
commit 9c4105b
Show file tree

Hide file tree

Showing 10 changed files with 20 additions and 16 deletions.
diff --git a/http_blaster.go b/http_blaster.go
@@ -135,7 +135,7 @@ func generate_executors() {
  for Name, workload := range cfg.Workloads {
  log.Println("Adding executor for ", Name)
  workload.Id = get_workload_id()
- e := &httpblaster.Executor{Workload: workload, Host: cfg.Global.Server,
+ e := &httpblaster.Executor{Global:cfg.Global, Workload: workload, Host: cfg.Global.Server,
  Port: cfg.Global.Port, TLS_mode: cfg.Global.TLSMode,
  StatusCodesAcceptance: cfg.Global.StatusCodesAcceptance, Data_bfr: dataBfr}
  executors = append(executors, e)

diff --git a/httpblaster/config/config.go b/httpblaster/config/config.go
@@ -49,17 +49,18 @@ func (r *Sep) UnmarshalText(text []byte) error {
 
 type TomlConfig struct {
  Title string
- Global global
+ Global Global
  Workloads map[string]Workload
 }
 
-type global struct {
+type Global struct {
  Duration duration
  Block_size int32
  Server string
  Port string
  TLSMode bool
  StatusCodesAcceptance map[string]float64
+ IgnoreAttrs []string
 }
 
 type Workload struct {

diff --git a/httpblaster/executor.go b/httpblaster/executor.go
@@ -48,6 +48,7 @@ type executor_result struct {
 type Executor struct {
  connections int32
  Workload config.Workload
+ Global config.Global
  Host string
  Port string
  TLS_mode bool
@@ -85,7 +86,7 @@ func (self *Executor) load_request_generator() (chan *fasthttp.Request, bool) {
  default:
  panic(fmt.Sprintf("unknown request generator %s", self.Workload.Generator))
  }
- ch_req := req_gen.GenerateRequests(self.Workload, self.TLS_mode, self.Host)
+ ch_req := req_gen.GenerateRequests(self.Global, self.Workload, self.TLS_mode, self.Host)
  return ch_req, release_req
 }
 

diff --git a/httpblaster/request_generators/csv2kv.go b/httpblaster/request_generators/csv2kv.go
@@ -87,7 +87,7 @@ func (self *Csv2KV) generate(ch_req chan *fasthttp.Request, payload string, host
  wg.Wait()
 }
 
-func (self *Csv2KV) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *Csv2KV) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  //panic(fmt.Sprintf("workload key [%s] workload key sep [%s]", wl.KeyFormat, string(wl.KeyFormatSep.Rune)))
  if self.workload.Header == nil {

diff --git a/httpblaster/request_generators/generator_interface.go b/httpblaster/request_generators/generator_interface.go
@@ -7,5 +7,5 @@ import (
 
 type Generator interface {
  UseCommon(c RequestCommon)
- GenerateRequests(workload config.Workload, tls_mode bool, host string) chan *fasthttp.Request
+ GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request
 }
diff --git a/httpblaster/request_generators/json2kv.go b/httpblaster/request_generators/json2kv.go
@@ -78,7 +78,7 @@ func (self *Json2KV) generate(ch_req chan *fasthttp.Request, payload string, hos
  wg.Wait()
 }
 
-func (self *Json2KV) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *Json2KV) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  //panic(fmt.Sprintf("workload key [%s] workload key sep [%s]", wl.KeyFormat, string(wl.KeyFormatSep.Rune)))
  if self.workload.Header == nil {

diff --git a/httpblaster/request_generators/line2kv.go b/httpblaster/request_generators/line2kv.go
@@ -77,7 +77,7 @@ func (self *Line2KvGenerator) generate(ch_req chan *fasthttp.Request, payload st
  log.Println("generators done")
 }
 
-func (self *Line2KvGenerator) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *Line2KvGenerator) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  if self.workload.Header == nil {
  self.workload.Header = make(map[string]string)

diff --git a/httpblaster/request_generators/line2stream.go b/httpblaster/request_generators/line2stream.go
@@ -78,7 +78,7 @@ func (self *Line2StreamGenerator) generate(ch_req chan *fasthttp.Request, payloa
  log.Println("generators done")
 }
 
-func (self *Line2StreamGenerator) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *Line2StreamGenerator) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  if self.workload.Header == nil {
  self.workload.Header = make(map[string]string)

diff --git a/httpblaster/request_generators/performance_generator.go b/httpblaster/request_generators/performance_generator.go
@@ -21,7 +21,7 @@ func (self *PerformanceGenerator) UseCommon(c RequestCommon) {
 
 }
 
-func (self *PerformanceGenerator) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *PerformanceGenerator) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  self.SetBaseUri(tls_mode, host, self.workload.Container, self.workload.Target)
  var contentType string = "text/html"

diff --git a/httpblaster/request_generators/restore.go b/httpblaster/request_generators/restore.go
@@ -24,7 +24,7 @@ type RestoreGenerator struct {
  re_name *regexp.Regexp
  re_collection_id *regexp.Regexp
  re_remove_items *regexp.Regexp
- emd_attrs  []string
+ emd_ignore_attrs []string
 }
 
 type BackupItem struct {
@@ -88,7 +88,7 @@ func (self *RestoreGenerator) generate_items(ch_lines chan []byte, collection_id
  item_name := i["__name"]["S"]
  collection_id := i["__collection_id"]["N"]
  dir_name := collection_ids[collection_id]
- for _, attr := range self.emd_attrs {
+ for _, attr := range self.emd_ignore_attrs {
  delete(i, attr)
  }
 
@@ -168,16 +168,18 @@ func (self *RestoreGenerator) line_reader() chan []byte {
  return ch_lines
 }
 
-func (self *RestoreGenerator) GenerateRequests(wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
+func (self *RestoreGenerator) GenerateRequests(global config.Global, wl config.Workload, tls_mode bool, host string) chan *fasthttp.Request {
  self.workload = wl
  ch_req := make(chan *fasthttp.Request, 100000)
 
  if self.workload.Header == nil {
  self.workload.Header = make(map[string]string)
  }
- self.emd_attrs = []string{`__name`, `__size`, `__atime_secs`, `__mtime_secs`, `__ctime_secs`, `__atime_nsecs`,
- `__mtime_nsecs`, `__ctime_nsecs`, `__inode_number`, `__obj_type`, `__collection_id`,
- `__tiny_low`, `__tiny_high`, `__uid`, `__gid`, `__mode`}
+ //self.emd_attrs = []string{`__name`, `__size`, `__atime_secs`, `__mtime_secs`, `__ctime_secs`, `__atime_nsecs`,
+ // `__mtime_nsecs`, `__ctime_nsecs`, `__inode_number`, `__obj_type`, `__collection_id`,
+ // `__tiny_low`, `__tiny_high`, `__uid`, `__gid`, `__mode`}
+ self.emd_ignore_attrs = global.IgnoreAttrs
+
  self.workload.Header["X-v3io-function"] = "PutItem"
 
  self.SetBaseUri(tls_mode, host, self.workload.Container, self.workload.Target)