Implement the Vitess Structure Logger VTSLoger

Signed-off-by: Emad Habib <ehabib@slack-corp.com>

go/flags/endtoend/mysqlctl.txt

@@ -88,6 +88,7 @@ Flags:
       --service_map strings                                         comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice
       --socket_file string                                          Local unix socket file to listen on
       --stderrthreshold severityFlag                                logs at or above this threshold go to stderr (default 1)
+      --structured-logging                                          whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                  interval in milliseconds to refresh tables in status page with refreshRequired class
       --tablet_dir string                                           The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.
       --tablet_uid uint32                                           Tablet UID. (default 41983)

go/flags/endtoend/mysqlctld.txt

@@ -114,6 +114,7 @@ Flags:
       --shutdown-wait-time duration                                      How long to wait for mysqld shutdown (default 5m0s)
       --socket_file string                                               Local unix socket file to listen on
       --stderrthreshold severityFlag                                     logs at or above this threshold go to stderr (default 1)
+      --structured-logging                                               whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                       interval in milliseconds to refresh tables in status page with refreshRequired class
       --tablet_dir string                                                The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.
       --tablet_uid uint32                                                Tablet UID (default 41983)

go/flags/endtoend/vtcombo.txt

@@ -334,6 +334,7 @@ Flags:
       --stderrthreshold severityFlag                                     logs at or above this threshold go to stderr (default 1)
       --stream_buffer_size int                                           the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768)
       --stream_health_buffer_size uint                                   max streaming health entries to buffer per streaming health client (default 20)
+      --structured-logging                                               whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                       interval in milliseconds to refresh tables in status page with refreshRequired class
       --table_gc_lifecycle string                                        States for a DROP TABLE garbage collection cycle. Default is 'hold,purge,evac,drop', use any subset ('drop' implicitly always included) (default "hold,purge,evac,drop")
       --tablet_dir string                                                The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.

go/flags/endtoend/vtctld.txt

@@ -130,7 +130,7 @@ Flags:
       --stats_common_tags strings                                        Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2
       --stats_drop_variables string                                      Variables to be dropped from the list of exported variables.
       --stats_emit_period duration                                       Interval between emitting stats to all registered backends (default 1m0s)
-      --stderrthreshold severityFlag                                     logs at or above this threshold go to stderr (default 1)
+      --structured-logging                                               whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                       interval in milliseconds to refresh tables in status page with refreshRequired class
       --tablet_dir string                                                The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.
       --tablet_grpc_ca string                                            the server ca to use to validate servers when connecting

go/flags/endtoend/vtgate.txt

@@ -192,6 +192,7 @@ Flags:
       --statsd_sample_rate float                                         Sample rate for statsd metrics (default 1)
       --stderrthreshold severityFlag                                     logs at or above this threshold go to stderr (default 1)
       --stream_buffer_size int                                           the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768)
+      --structured-logging                                               whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                       interval in milliseconds to refresh tables in status page with refreshRequired class
       --tablet_filters strings                                           Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch.
       --tablet_grpc_ca string                                            the server ca to use to validate servers when connecting

go/flags/endtoend/vtorc.txt

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 VTOrc is the automated fault detection and repair tool in Vitess.
 
 Usage:
@@ -78,6 +79,7 @@ Flags:
       --stats_drop_variables string                                 Variables to be dropped from the list of exported variables.
       --stats_emit_period duration                                  Interval between emitting stats to all registered backends (default 1m0s)
       --stderrthreshold severityFlag                                logs at or above this threshold go to stderr (default 1)
+      --structured-logging                                          whether to use structured logging (Zap) or the original (glog) logger
       --table-refresh-interval int                                  interval in milliseconds to refresh tables in status page with refreshRequired class
       --tablet_manager_grpc_ca string                               the server ca to use to validate servers when connecting
       --tablet_manager_grpc_cert string                             the cert to use to connect
@@ -113,3 +115,93 @@ Flags:
   -v, --version                                                     print binary version
       --vmodule vModuleFlag                                         comma-separated list of pattern=N settings for file-filtered logging
       --wait-replicas-timeout duration                              Duration for which to wait for replica's to respond when issuing RPCs (default 30s)
+=======
+Usage of vtorc:
+      --alsologtostderr                              log to standard error as well as files
+      --audit-file-location string                   File location where the audit logs are to be stored
+      --audit-purge-duration duration                Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s)
+      --audit-to-backend                             Whether to store the audit log in the VTOrc database
+      --audit-to-syslog                              Whether to store the audit log in the syslog
+      --catch-sigpipe                                catch and ignore SIGPIPE on stdout and stderr if specified
+      --clusters_to_watch strings                    Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80"
+      --config string                                config file name
+      --consul_auth_static_file string               JSON File to read the topos/tokens from.
+      --emit_stats                                   If set, emit stats to push-based monitoring and stats backends
+      --grpc_auth_static_client_creds string         When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
+      --grpc_compression string                      Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy
+      --grpc_enable_tracing                          Enable gRPC tracing.
+      --grpc_initial_conn_window_size int            gRPC initial connection window size
+      --grpc_initial_window_size int                 gRPC initial window size
+      --grpc_keepalive_time duration                 After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s)
+      --grpc_keepalive_timeout duration              After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
+      --grpc_max_message_size int                    Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
+      --grpc_prometheus                              Enable gRPC monitoring with Prometheus.
+  -h, --help                                         display usage and exit
+      --instance-poll-time duration                  Timer duration on which VTOrc refreshes MySQL information (default 5s)
+      --keep_logs duration                           keep logs for this long (using ctime) (zero to keep forever)
+      --keep_logs_by_mtime duration                  keep logs for this long (using mtime) (zero to keep forever)
+      --lameduck-period duration                     keep running at least this long after SIGTERM before stopping (default 50ms)
+      --lock-timeout duration                        Maximum time for which a shard/keyspace lock can be acquired for (default 45s)
+      --log_backtrace_at traceLocation               when logging hits line file:N, emit a stack trace (default :0)
+      --log_dir string                               If non-empty, write log files in this directory
+      --log_err_stacks                               log stack traces for errors
+      --log_rotate_max_size uint                     size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800)
+      --logtostderr                                  log to standard error instead of files
+      --max-stack-size int                           configure the maximum stack size in bytes (default 67108864)
+      --onclose_timeout duration                     wait no more than this for OnClose handlers before stopping (default 10s)
+      --onterm_timeout duration                      wait no more than this for OnTermSync handlers before stopping (default 10s)
+      --pid_file string                              If set, the process will write its pid to the named file, and delete it on graceful shutdown.
+      --port int                                     port for the server
+      --pprof strings                                enable profiling
+      --prevent-cross-cell-failover                  Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover
+      --purge_logs_interval duration                 how often try to remove old logs (default 1h0m0s)
+      --reasonable-replication-lag duration          Maximum replication lag on replicas which is deemed to be acceptable (default 10s)
+      --recovery-period-block-duration duration      Duration for which a new recovery is blocked on an instance after running a recovery (default 30s)
+      --recovery-poll-duration duration              Timer duration on which VTOrc polls its database to run a recovery (default 1s)
+      --remote_operation_timeout duration            time to wait for a remote operation (default 15s)
+      --security_policy string                       the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
+      --shutdown_wait_time duration                  Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s)
+      --snapshot-topology-interval duration          Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours
+      --sqlite-data-file string                      SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared")
+      --stats_backend string                         The name of the registered push-based monitoring/stats backend to use
+      --stats_combine_dimensions string              List of dimensions to be combined into a single "all" value in exported stats vars
+      --stats_common_tags strings                    Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2
+      --stats_drop_variables string                  Variables to be dropped from the list of exported variables.
+      --stats_emit_period duration                   Interval between emitting stats to all registered backends (default 1m0s)
+      --stderrthreshold severity                     logs at or above this threshold go to stderr (default 1)
+      --structured-logging                           whether to use structured logging (Zap) or the original (glog) logger
+      --table-refresh-interval int                   interval in milliseconds to refresh tables in status page with refreshRequired class
+      --tablet_manager_grpc_ca string                the server ca to use to validate servers when connecting
+      --tablet_manager_grpc_cert string              the cert to use to connect
+      --tablet_manager_grpc_concurrency int          concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8)
+      --tablet_manager_grpc_connpool_size int        number of tablets to keep tmclient connections open to (default 100)
+      --tablet_manager_grpc_crl string               the server crl to use to validate server certificates when connecting
+      --tablet_manager_grpc_key string               the key to use to connect
+      --tablet_manager_grpc_server_name string       the server name to use to validate server certificate
+      --tablet_manager_protocol string               Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc")
+      --topo-information-refresh-duration duration   Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s)
+      --topo_consul_lock_delay duration              LockDelay for consul session. (default 15s)
+      --topo_consul_lock_session_checks string       List of checks for consul session. (default "serfHealth")
+      --topo_consul_lock_session_ttl string          TTL for consul session.
+      --topo_consul_watch_poll_duration duration     time of the long poll for watch queries. (default 30s)
+      --topo_etcd_lease_ttl int                      Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
+      --topo_etcd_tls_ca string                      path to the ca to use to validate the server cert when connecting to the etcd topo server
+      --topo_etcd_tls_cert string                    path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS
+      --topo_etcd_tls_key string                     path to the client key to use to connect to the etcd topo server, enables TLS
+      --topo_global_root string                      the path of the global topology data in the global topology server
+      --topo_global_server_address string            the address of the global topology server
+      --topo_implementation string                   the topology implementation to use
+      --topo_k8s_context string                      The kubeconfig context to use, overrides the 'current-context' from the config
+      --topo_k8s_kubeconfig string                   Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod
+      --topo_k8s_namespace string                    The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config
+      --topo_zk_auth_file string                     auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
+      --topo_zk_base_timeout duration                zk base timeout (see zk.Connect) (default 30s)
+      --topo_zk_max_concurrency int                  maximum number of pending requests to send to a Zookeeper server. (default 64)
+      --topo_zk_tls_ca string                        the server ca to use to validate servers when connecting to the zk topo server
+      --topo_zk_tls_cert string                      the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS
+      --topo_zk_tls_key string                       the key to use to connect to the zk topo server, enables TLS
+      --v Level                                      log level for V logs
+  -v, --version                                      print binary version
+      --vmodule moduleSpec                           comma-separated list of pattern=N settings for file-filtered logging
+      --wait-replicas-timeout duration               Duration for which to wait for replica's to respond when issuing RPCs (default 30s)
+>>>>>>> 1540296507 (Implement the Vitess Structure Logger VTSLoger)

go/flags/endtoend/vttablet.txt

@@ -332,6 +332,7 @@ Flags:
       --statsd_sample_rate float                                         Sample rate for statsd metrics (default 1)
       --stderrthreshold severityFlag                                     logs at or above this threshold go to stderr (default 1)
       --stream_health_buffer_size uint                                   max streaming health entries to buffer per streaming health client (default 20)
+      --structured-logging                                               whether to use structured logging (Zap) or the original (glog) logger
       --table-acl-config string                                          path to table access checker config file; send SIGHUP to reload this file
       --table-acl-config-reload-interval duration                        Ticker to reload ACLs. Duration flag, format e.g.: 30s. Default: do not reload
       --table-refresh-interval int                                       interval in milliseconds to refresh tables in status page with refreshRequired class