Merge branch 'master' into stable

.circleci/config.yml

@@ -33,6 +33,8 @@ jobs:
       - run: cd $HOME && for version in $VERSIONS; do mkdir -p go_path_$version/src/github.com/cossacklabs/themis/gothemis; mkdir -p go_path_$version/src/github.com/cossacklabs/acra; rsync -auv $HOME/themis/gothemis/ go_path_$version/src/github.com/cossacklabs/themis/gothemis; rsync -auv $HOME/project/ go_path_$version/src/github.com/cossacklabs/acra; done
       - run: cd $HOME && for version in $VERSIONS; do GOROOT=$HOME/go_root_$version/go PATH=$GOROOT/bin/:$PATH GOPATH=$HOME/go_path_$version go get -d github.com/cossacklabs/acra/...; done
       - run: pip3 install -r $HOME/project/tests/requirements.txt
+      # install from sources because pip install git+https://github.com/mysql/mysql-connector-python not support recursive submodules
+      - run: git clone https://github.com/Lagovas/mysql-connector-python; cd mysql-connector-python; sudo python3 setup.py clean build_py install_lib
       - run: sudo ldconfig
     # testing
       # delete file if exists

CHANGELOG.md

@@ -1,5 +1,59 @@
 # Acra ChangeLog
 
+## [0.81.0](https://github.com/cossacklabs/acra/releases/tag/0.81), July 6th 2018
+
+_Core_:
+
+- **Prepared Statements**
+
+   - Added support of prepared statements for PostgreSQL/MySQL. Both binary and text response formats are supported ([#192](https://github.com/cossacklabs/acra/pull/192)).
+
+- **SQL requests filtering in AcraCensor**
+
+   AcraCensor got smarter in preventing SQL Injections.
+
+   - Improved flexibility for parsing queries. If AcraCensor can't parse an SQL query, it is considered as potentially too dangerous to send it to the database, so AcraCensor blocks such "unparseable" queries by default. 
+   However, setting the configuration flag `ignore_parse_error` to `true` will make AcraCensor ignore the "unparseable" quality of queries and send them to the database anyway. Check out the configuration example in [configs/acra-censor.example.yaml](https://github.com/cossacklabs/acra/blob/master/configs/acra-censor.example.yaml) ([#194](https://github.com/cossacklabs/acra/pull/194)).
+
+   - Added support of complex JOIN queries ([#191](https://github.com/cossacklabs/acra/pull/191)).
+
+   - Improved reading/writing QueryCapture log file. Now AcraCensor uses bufferisation before writing queries into a log file. 
+   Changed format of QueryCapture log to JSON Line (each query sits in a separate line in a log file instead of having an array of JSON objects) ([#193](https://github.com/cossacklabs/acra/pull/193)).
+
+   - Introduced a few fixes here and there, made integration tests for AcraCensor more stable ([#184](https://github.com/cossacklabs/acra/pull/184)).
+
+- **Improving MySQL support**
+
+   We introduced MySQL support just a few Acra releases ago and we continue polishing it. Now we've updated the example projects so you can jump right into the code!
+
+   Take a look at how to use Acra for both PostreSQL and MySQL databases in these examples:
+
+   - Go: see the [examples/golang](https://github.com/cossacklabs/acra/tree/master/examples/golang/src) folder ([#190](https://github.com/cossacklabs/acra/pull/190)).
+
+   - Ruby: see the [examples/ruby](https://github.com/cossacklabs/acra/tree/master/examples/ruby) folder ([#189](https://github.com/cossacklabs/acra/pull/189)).
+
+   - Python: see the [examples/python](https://github.com/cossacklabs/acra/tree/master/examples/python) folder ([#188](https://github.com/cossacklabs/acra/pull/188)).
+
+- **Other**
+
+   - Updated handling of message formats for PostgreSQL and MySQL protocols ([#186](https://github.com/cossacklabs/acra/pull/186)).
+
+   - Improved logging in CEF and JSON formats for high load systems ([#195](https://github.com/cossacklabs/acra/pull/195)).
+
+   - Added comprehensive `Readme` to every project in [/examples](https://github.com/cossacklabs/acra/tree/master/examples) folder ([#196](https://github.com/cossacklabs/acra/pull/196)).
+
+   - Added pre-generated configuration file for AcraAuthmanager. Now it's easier to configure AcraServer using [AcraWebconfig](https://github.com/cossacklabs/acra/wiki/AcraWebConfig) ([#187](https://github.com/cossacklabs/acra/pull/187)).
+
+
+_Documentation_:
+
+- Updated Acra [Architecture and Data flow](https://github.com/cossacklabs/acra/wiki/Architecture-and-data-flow) graphic schemes to better illustrate Acra's components, connections between them, and typical use-cases.
+- Updated AcraCensor's description to explain how [unparseable queries](https://github.com/cossacklabs/acra/wiki/AcraCensor#unparseable-queries) are handled.    
+- Described typical [Public Key Infrastructure](https://github.com/cossacklabs/acra/wiki/PKI-overview-for-Acra) with some advice on where to put Acra in the general scheme of things.
+- Described Acra's [Security Model](https://github.com/cossacklabs/acra/wiki/Acra's-Security-Model), possible threats, and possible consequences of compromisation.
+- Added a page describing the ways [Acra can help you better comply with GDPR](https://github.com/cossacklabs/acra/wiki/Acra-and-GDPR-compliance).
+
+
 ## [0.80.0](https://github.com/cossacklabs/acra/releases/tag/0.80), May 31st 2018
 
 _Core_:
@@ -26,7 +80,7 @@ _Core_:
    | acra_rollback | AcraRollback | decrypts the whole database |
    | acra_genkeys | AcraKeymaker | generates encryption keys for storage and transport of the Acra components |
    | acra_genauth | AcraAuthmanager | generates user accounts for AcraWebConfig |
-   | acra_genpoisonrecord | AcraPoisonRecordMaker | generates poision records for databases |
+   | acra_genpoisonrecord | AcraPoisonRecordMaker | generates poison records for databases |
    | acra_addzone | AcraAddzone | generates Zones' header for AcraWriter |
 
    Check the configurations of components inside [/configs folder](https://github.com/cossacklabs/acra/tree/master/configs) and read [Migration Guide](https://github.com/cossacklabs/acra/wiki/Migration-guide) for more details ([#175](https://github.com/cossacklabs/acra/pull/175), [#174](https://github.com/cossacklabs/acra/pull/174), [#173](https://github.com/cossacklabs/acra/pull/173), [#170](https://github.com/cossacklabs/acra/pull/170), [#169](https://github.com/cossacklabs/acra/pull/169), [#168](https://github.com/cossacklabs/acra/pull/168)).

acra-censor/acra-censor_configuration_provider.go

@@ -8,7 +8,8 @@ import (
 
 const BlacklistConfigStr = "blacklist"
 const WhitelistConfigStr = "whitelist"
-const LoggerConfigStr = "query_capture"
+const QueryCaptureConfigStr = "query_capture"
+const QueryIgnoreConfigStr = "query_ignore"
 
 type AcraCensorConfig struct {
 	Handlers []struct {
@@ -18,60 +19,62 @@ type AcraCensorConfig struct {
 		Rules    []string
 		Filepath string
 	}
+	IgnoreParseError bool `yaml:"ignore_parse_error"`
 }
 
-func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) ([]QueryHandlerInterface, error) {
-	var handlers_ []QueryHandlerInterface
-
+func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) error {
 	var censorConfiguration AcraCensorConfig
 	err := yaml.Unmarshal(configuration, &censorConfiguration)
 	if err != nil {
-		return nil, err
+		return err
 	}
+	acraCensor.ignoreParseError = censorConfiguration.IgnoreParseError
 	for _, handlerConfiguration := range censorConfiguration.Handlers {
 		switch handlerConfiguration.Handler {
 		case WhitelistConfigStr:
 			whitelistHandler := &handlers.WhitelistHandler{}
 			err := whitelistHandler.AddQueries(handlerConfiguration.Queries)
 			if err != nil {
-				return nil, err
+				return err
 			}
 			whitelistHandler.AddTables(handlerConfiguration.Tables)
 			err = whitelistHandler.AddRules(handlerConfiguration.Rules)
 			if err != nil {
-				return nil, err
+				return err
 			}
 			acraCensor.AddHandler(whitelistHandler)
-			handlers_ = append(handlers_, whitelistHandler)
 			break
 		case BlacklistConfigStr:
 			blacklistHandler := &handlers.BlacklistHandler{}
 			err := blacklistHandler.AddQueries(handlerConfiguration.Queries)
 			if err != nil {
-				return nil, err
+				return err
 			}
 			blacklistHandler.AddTables(handlerConfiguration.Tables)
 			err = blacklistHandler.AddRules(handlerConfiguration.Rules)
 			if err != nil {
-				return nil, err
+				return err
 			}
 			acraCensor.AddHandler(blacklistHandler)
-			handlers_ = append(handlers_, blacklistHandler)
 			break
-		case LoggerConfigStr:
+		case QueryCaptureConfigStr:
 			if strings.EqualFold(handlerConfiguration.Filepath, "") {
 				break
 			}
-			logger, err := handlers.NewQueryCaptureHandler(handlerConfiguration.Filepath)
+			queryCaptureHandler, err := handlers.NewQueryCaptureHandler(handlerConfiguration.Filepath)
 			if err != nil {
-				return nil, err
+				return err
 			}
-			acraCensor.AddHandler(logger)
-			handlers_ = append(handlers_, logger)
+			acraCensor.AddHandler(queryCaptureHandler)
+			break
+		case QueryIgnoreConfigStr:
+			queryIgnoreHandler := handlers.NewQueryIgnoreHandler()
+			queryIgnoreHandler.AddQueries(handlerConfiguration.Queries)
+			acraCensor.AddHandler(queryIgnoreHandler)
 			break
 		default:
 			break
 		}
 	}
-	return handlers_, nil
+	return nil
 }

acra-censor/acra-censor_implementation.go

@@ -1,11 +1,13 @@
 package acracensor
 
 import (
+	"github.com/cossacklabs/acra/acra-censor/handlers"
 	log "github.com/sirupsen/logrus"
 )
 
 type AcraCensor struct {
-	handlers []QueryHandlerInterface
+	handlers         []QueryHandlerInterface
+	ignoreParseError bool
 }
 
 func (acraCensor *AcraCensor) AddHandler(handler QueryHandlerInterface) {
@@ -28,9 +30,18 @@ func (acraCensor *AcraCensor) ReleaseAll() {
 
 func (acraCensor *AcraCensor) HandleQuery(query string) error {
 	for _, handler := range acraCensor.handlers {
-		if err := handler.CheckQuery(query); err != nil {
+		continueHandling, err := handler.CheckQuery(query)
+		if err != nil {
+			if err == handlers.ErrQuerySyntaxError && acraCensor.ignoreParseError {
+				log.WithError(err).Infof("parsing error on query (first %v symbols): %s", handlers.LogQueryLength, handlers.TrimStringToN(query, handlers.LogQueryLength))
+				continue
+			}
 			log.Errorf("Forbidden query: '%s'", query)
 			return err
+		} else {
+			if !continueHandling {
+				return nil
+			}
 		}
 	}
 	log.Infof("Allowed query: '%s'", query)

acra-censor/acra-censor_interfaces.go

@@ -1,8 +1,7 @@
 package acracensor
 
 type QueryHandlerInterface interface {
-	CheckQuery(sqlQuery string) error
-	Reset()
+	CheckQuery(sqlQuery string) (bool, error) //1st return arg specifies whether continue verification or not, 2nd specifies whether query is forbidden
 	Release()
 }