diff --git a/.circleci/config.yml b/.circleci/config.yml index ea5fe2eaf..a1e9e50f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,20 +6,32 @@ jobs: # postgresql image with ssl support - image: nimbustech/postgres-ssl:9.5 environment: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres + POSTGRES_PASSWORD: test + POSTGRES_USER: test + POSTGRES_DB: test + # use the same credentials for mysql db as for postgresql (which support was added first) + # has latest tag on 2018.03.29 + - image: mysql:5.7.21 + environment: + MYSQL_DATABASE: test + MYSQL_USER: test + MYSQL_PASSWORD: test + MYSQL_ROOT_PASSWORD: root environment: GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis FILEPATH_ERROR_FLAG: /tmp/test_fail - VERSIONS: 1.6 1.6.4 1.7 1.7.5 1.8 1.9.4 1.10 + VERSIONS: 1.7 1.7.6 1.8 1.8.7 1.9.4 1.10 + TEST_DB_USER: test + TEST_DB_USER_PASSWORD: test + TEST_DB_NAME: test steps: # prepare - - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync + - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync psmisc - run: cd $HOME && git clone https://github.com/cossacklabs/themis && cd themis && sudo make install - run: cd $HOME && for version in $VERSIONS; do mkdir go_root_$version; cd go_root_$version; wget https://storage.googleapis.com/golang/go$version.linux-amd64.tar.gz; tar xf go$version.linux-amd64.tar.gz; cd -; done - checkout - run: cd $HOME && for version in $VERSIONS; do mkdir -p go_path_$version/src/github.com/cossacklabs/themis/gothemis; mkdir -p go_path_$version/src/github.com/cossacklabs/acra; rsync -auv $HOME/themis/gothemis/ go_path_$version/src/github.com/cossacklabs/themis/gothemis; rsync -auv $HOME/project/ go_path_$version/src/github.com/cossacklabs/acra; done - - run: cd $HOME && for version in $VERSIONS; do GOROOT=$HOME/go_root_$version/go PATH=$GOROOT/bin/:$PATH GOPATH=$HOME/go_path_$version go get github.com/cossacklabs/acra/...; done + - run: cd $HOME && for version in $VERSIONS; do GOROOT=$HOME/go_root_$version/go PATH=$GOROOT/bin/:$PATH GOPATH=$HOME/go_path_$version go get -d github.com/cossacklabs/acra/...; done - run: pip3 install -r $HOME/project/tests/requirements.txt - run: sudo ldconfig # testing diff --git a/.circleci/integration.sh b/.circleci/integration.sh index 39d30ab92..b41c8fd7c 100755 --- a/.circleci/integration.sh +++ b/.circleci/integration.sh @@ -1,23 +1,59 @@ #!/usr/bin/env bash + export TEST_ACRA_PORT=6000 export TEST_PROXY_PORT=7000 export TEST_PROXY_COMMAND_PORT=8000 cd $HOME/project for version in $VERSIONS; do -export TEST_ACRA_PORT=$(expr ${TEST_ACRA_PORT} + 1); -export TEST_PROXY_PORT=$(expr ${TEST_PROXY_PORT} + 1); -export TEST_PROXY_COMMAND_PORT=$(expr ${TEST_PROXY_COMMAND_PORT} + 1); -export GOROOT=$HOME/go_root_$version/go; -export PATH=$GOROOT/bin/:$PATH; -export GOPATH=$HOME/go_path_$version; - -export TEST_TLS=on -python3 tests/test.py; -if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG"; -fi - -export TEST_TLS=off -python3 tests/test.py; -if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG"; -fi + echo "-------------------- Testing Go version $version" + + export TEST_ACRA_PORT=$(expr ${TEST_ACRA_PORT} + 1); + export TEST_PROXY_PORT=$(expr ${TEST_PROXY_PORT} + 1); + export TEST_PROXY_COMMAND_PORT=$(expr ${TEST_PROXY_COMMAND_PORT} + 1); + export GOROOT=$HOME/go_root_$version/go; + export PATH=$GOROOT/bin/:$PATH; + export GOPATH=$HOME/go_path_$version; + + # setup postgresql credentials + #export TEST_DB_USER=${POSTGRES_USER} + #export TEST_DB_USER_PASSWORD=${POSTGRES_PASSWORD} + #export TEST_DB_NAME=postgres + export TEST_DB_PORT=5432 + unset TEST_MYSQL + + export TEST_TLS=on + + echo "-------------------- Testing POSTGRES with TEST_TLS=on" + + python3 tests/test.py -v; + if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG"; + fi + + export TEST_TLS=off + + echo "-------------------- Testing POSTGRES with TEST_TLS=off" + python3 tests/test.py -v; + if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG"; + fi + + # setup mysql credentials + #export TEST_DB_USER=${MYSQL_USER} + #export TEST_DB_USER_PASSWORD=${MYSQL_PASSWORD} + #export TEST_DB_NAME=${MYSQL_DATABASE} + export TEST_DB_PORT=3306 + export TEST_MYSQL=true + + + echo "-------------------- Testing TEST_MYSQL with TEST_TLS=off" + export TEST_TLS=off + python3 tests/test.py -v; + if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG"; + fi + + echo "-------------------- Testing TEST_MYSQL with TEST_TLS=on" + export TEST_TLS=on + python3 tests/test.py -v; + if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG"; + fi + done diff --git a/.gitignore b/.gitignore index db4645178..015ee75ab 100644 --- a/.gitignore +++ b/.gitignore @@ -117,3 +117,5 @@ ENV/ # Rope project settings .ropeproject + +cmd/acra_configui/auth.keys diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d0e4757..ace5dfa59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,92 @@ # Acra ChangeLog +## [0.77.0](https://github.com/cossacklabs/acra/releases/tag/0.77), April 13th 2018 + + +_Core_: + +- **MySQL databases** + + - Added support for MySQL: now you can connect Acra to MySQL databases. Works with any SSL mode: `require`, `allow`, `disable`. + - Tested and supported on: MySQL ([#155](https://github.com/cossacklabs/acra/pull/155), [#140](https://github.com/cossacklabs/acra/pull/140)). + + > Note: Prepared statements are not supported yet, but this feature is coming soon! + + Read about the new configurations on the [AcraServer](https://github.com/cossacklabs/acra/wiki/How-AcraServer-works) documentation page. + +- **Keeping keys in secret** + + - Added encryption for the keys' folder: private keys are now symmetrically encrypted by `master_key` ([#143](https://github.com/cossacklabs/acra/pull/143)) for storage. + - Added ability to generate public/private keys in the separate folders ([#148](https://github.com/cossacklabs/acra/pull/148), [#142](https://github.com/cossacklabs/acra/pull/142)). + + Read more about the current changes in [key management here](https://github.com/cossacklabs/acra/wiki/Key-Management). + +- **Filtering requests for MySQL** + + - Added firewall component named [AcraCensor](https://github.com/cossacklabs/acra/wiki/acracensor) to handle MySQL queries.
+ You can provide a list of restricted or allowed tables, columns, and exact queries to handle. AcraCensor will pass the allowed queries and return error on forbidden ones. Rules are configured and stored in `yaml` file. Each request is logged in real time. Moreover, all the queries and their states are logged into a separate log file. ([#151](https://github.com/cossacklabs/acra/pull/151), [#138](https://github.com/cossacklabs/acra/pull/138), [#136](https://github.com/cossacklabs/acra/pull/136), [#132](https://github.com/cossacklabs/acra/pull/132), [#125](https://github.com/cossacklabs/acra/pull/125), [#108](https://github.com/cossacklabs/acra/pull/108)).
+ + See a detailed description of AcraCensor on the corresponding [AcraCensor documentation page](https://github.com/cossacklabs/acra/wiki/acracensor). + +- **Web Config UI** + + - Added lightweight HTTP [web server](https://github.com/cossacklabs/acra/wiki/AcraConfigUI) for managing AcraServer's certain configuration options.
+ You can update the proxy address and port, database address and port, handling of Zone mode and poison records. On saving new configuration, `acraserver` will gracefully restart and use these settings automatically. The access to thiw web page is restricted using basic auth. ([#153](https://github.com/cossacklabs/acra/pull/153), [#141](https://github.com/cossacklabs/acra/pull/141), [#123](https://github.com/cossacklabs/acra/pull/123), [#111](https://github.com/cossacklabs/acra/pull/111)).
+ + See the interface screenshot and detailed instructions at [Acra Config UI](https://github.com/cossacklabs/acra/wiki/AcraConfigUI) page. + + +- **Logging** + - Added support of new logging formats: plaintext, [CEF](https://kc.mcafee.com/resources/sites/MCAFEE/content/live/CORP_KNOWLEDGEBASE/78000/KB78712/en_US/CEF_White_Paper_20100722.pdf), and json.
+ Logging mode and verbosity level is configured for AcraServer, AcraProxy, and AcraConfigUI in the corresponding `yaml` files. Log messages were slightly improved, custom error codes were added (which we believe will help to understand and debug any issues) ([#135](https://github.com/cossacklabs/acra/pull/135), [#126](https://github.com/cossacklabs/acra/pull/126), [#110](https://github.com/cossacklabs/acra/pull/110)). + + Read more about the log analysis at [Logging](https://github.com/cossacklabs/acra/wiki/Logging) page. + + +- **Tests** + + - Added many new integartion tests, fixed stability and handling of more complicated use-cases ([#150](https://github.com/cossacklabs/acra/pull/150), [#147](https://github.com/cossacklabs/acra/pull/147), [#137](https://github.com/cossacklabs/acra/pull/137), [#117](https://github.com/cossacklabs/acra/pull/117), [#116](https://github.com/cossacklabs/acra/pull/116), [#115](https://github.com/cossacklabs/acra/pull/115)). + + +_Infrastructure_: + +- **Docker support** + + - Added Docker Container for every main component: `AcraServer`, `AcraProxy`, `AcraConfigUI`, and key generators (`AcraGenKeys` and `AcraGenAuth`). You can find the containers in [/docker](https://github.com/cossacklabs/acra/tree/master/docker) folder or on the [Docker Hub](https://hub.docker.com/r/cossacklabs/) ([#139](https://github.com/cossacklabs/acra/pull/139)). + - Updated [Getting started with Docker](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker) guide to make starting out with Acra even easier. + + - Added easy-to-use docker-compose files to launch Acra in different environments, including key distribution. Possible configurations are: + - `acraserver` + `acra_configui `; + - connecting to PostreSQL or MySQL databases; + - using Secure Session or SSL as transport encryption; + - with or without `acraproxy`; + - with or without zones.
+ + This is huge! We encourage you to try it! Check out the instructions and examples in the [/docker](https://github.com/cossacklabs/acra/tree/master/docker) folder. ([#154](https://github.com/cossacklabs/acra/pull/154), [#146](https://github.com/cossacklabs/acra/pull/146), [#134](https://github.com/cossacklabs/acra/pull/134), [#133](https://github.com/cossacklabs/acra/pull/133), [#102](https://github.com/cossacklabs/acra/pull/102)). + +- **Go versions** + + - Updated the list of supported versions of Go. Every Acra component can now be built using Go >1.7, except `acra_rollback` that requires Go >1.8. No worries, you can still download Acra as a binary package anyway :) + +- **OS** + + - Dropped support of Debian Wheezy (no autotests, no precompiled binaries now). + + +_Documentation_: + +- Updated [QuickStart](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) documentation about launching and building Acra components. +- Described how to setup [AcraCensor](https://github.com/cossacklabs/acra/wiki/acracensor) and [AcraConfigUI](https://github.com/cossacklabs/acra/wiki/AcraConfigUI). +- Added more details and described new options (like using TLS and connecting to MySQL databases) for [AcraServer](https://github.com/cossacklabs/acra/wiki/How-AcraServer-works) and [AcraProxy](https://github.com/cossacklabs/acra/wiki/AcraProxy-and-AcraWriter). +- Described new [logging](https://github.com/cossacklabs/acra/wiki/Logging) formats. +- Updated description of [Key management](https://github.com/cossacklabs/acra/wiki/Key-Management) approach we encourage you to use. +- Described Docker components and ready-to-use Docker Compose configurations based on the [Docker Readme](https://github.com/cossacklabs/acra/tree/master/docker). +- Updated [Getting started with Docker](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker) guide. +- Distributed the information about master key across the docs. +- Many small improvements. + + + ## [0.76](https://github.com/cossacklabs/acra/releases/tag/0.76), March 9th 2018 diff --git a/Makefile b/Makefile index a47e5c997..cf59c790b 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,33 @@ ifneq ($(BUILD_PATH),) - BIN_PATH = $(BUILD_PATH) + BIN_PATH = $(BUILD_PATH) else - BIN_PATH = build + BIN_PATH = build endif -#default engine + ifeq ($(PREFIX),) -PREFIX = /usr + PREFIX = /usr endif + TEMP_GOPATH = temp_gopath -ABS_TEMP_GOPATH = $(shell pwd)/$(TEMP_GOPATH) +ABS_TEMP_GOPATH := $(shell pwd)/$(TEMP_GOPATH) + +ifneq ($(GIT_BRANCH),) + BRANCH = $(GIT_BRANCH) +else + BRANCH = master +endif GIT_VERSION := $(shell if [ -d ".git" ]; then git version; fi 2>/dev/null) ifdef GIT_VERSION - VERSION = $(shell git describe --tags HEAD | cut -b 1-) + VERSION = $(shell git describe --tags HEAD | cut -b 1-) + GIT_HASH = $(shell git rev-parse --verify HEAD) else - VERSION = $(shell date -I) + VERSION = $(shell date -I) endif +.PHONY: get_version dist temp_copy install clean test_go test_python test \ + test_all unpack_dist deb rpm docker docker_push + get_version: @echo $(VERSION) @@ -59,7 +70,6 @@ test: temp_copy test_go # alias for unification with other products test_all: test - PACKAGE_NAME = acra COSSACKLABS_URL = https://www.cossacklabs.com MAINTAINER = "Cossack Labs Limited " @@ -69,36 +79,37 @@ LICENSE_NAME = "Apache License Version 2.0" DEBIAN_CODENAME := $(shell lsb_release -cs 2> /dev/null) DEBIAN_ARCHITECTURE = `dpkg --print-architecture 2>/dev/null` -DEBIAN_DEPENDENCIES := --depends openssl --depends libthemis +DEBIAN_DEPENDENCIES = --depends openssl --depends libthemis RPM_DEPENDENCIES = --depends openssl --depends libthemis ifeq ($(shell lsb_release -is 2> /dev/null),Debian) - NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb - OS_CODENAME = $(shell lsb_release -cs) + NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb + OS_CODENAME = $(shell lsb_release -cs) else ifeq ($(shell lsb_release -is 2> /dev/null),Ubuntu) - NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb - OS_CODENAME = $(shell lsb_release -cs) + NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb + OS_CODENAME = $(shell lsb_release -cs) else - OS_NAME = $(shell cat /etc/os-release | grep -e "^ID=\".*\"" | cut -d'"' -f2) - OS_VERSION = $(shell cat /etc/os-release | grep -i version_id|cut -d'"' -f2) - ARCHITECTURE = $(shell arch) - RPM_VERSION = $(shell echo -n "$(VERSION)"|sed s/-/_/g) - NAME_SUFFIX = $(RPM_VERSION).$(OS_NAME)$(OS_VERSION).$(ARCHITECTURE).rpm + OS_NAME = $(shell cat /etc/os-release | grep -e "^ID=\".*\"" | cut -d'"' -f2) + OS_VERSION = $(shell cat /etc/os-release | grep -i version_id|cut -d'"' -f2) + ARCHITECTURE = $(shell arch) + RPM_VERSION = $(shell echo -n "$(VERSION)"|sed s/-/_/g) + NAME_SUFFIX = $(RPM_VERSION).$(OS_NAME)$(OS_VERSION).$(ARCHITECTURE).rpm endif SHORT_DESCRIPTION = "Acra helps you easily secure your databases in distributed, microservice-rich environments" RPM_SUMMARY = "Acra helps you easily secure your databases in distributed, microservice-rich environments. \ - It allows you to selectively encrypt sensitive records with strong multi-layer cryptography, detect potential \ - intrusions and SQL injections and cryptographically compartmentalize data stored in large sharded schemes. \ - Acra's security model guarantees that if your database or your application become compromised, they will not \ - leak sensitive data, or keys to decrypt them." + It allows you to selectively encrypt sensitive records with strong multi-layer cryptography, detect potential \ + intrusions and SQL injections and cryptographically compartmentalize data stored in large sharded schemes. \ + Acra's security model guarantees that if your database or your application become compromised, they will not \ + leak sensitive data, or keys to decrypt them." + +BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') unpack_dist: @tar -xf $(DIST_FILENAME) deb: install @mkdir -p '$(BIN_PATH)/deb' - @fpm --input-type dir \ --output-type deb \ --name $(PACKAGE_NAME) \ @@ -113,11 +124,9 @@ deb: install --deb-priority optional \ --category security \ $(TEMP_GOPATH)/bin/=$(PREFIX)/bin - # it's just for printing .deb files @find $(BIN_PATH) -name \*.deb - rpm: install @mkdir -p $(BIN_PATH)/rpm @fpm --input-type dir \ @@ -135,3 +144,42 @@ rpm: install $(TEMP_GOPATH)/bin/=$(PREFIX)/bin # it's just for printing .rpm files @find $(BIN_PATH) -name \*.rpm + +define docker_build + @docker image build \ + --no-cache=true \ + --build-arg VERSION=$(VERSION)\ + --build-arg VCS_URL="https://github.com/cossacklabs/acra" \ + --build-arg VCS_REF=$(GIT_HASH) \ + --build-arg VCS_BRANCH=$(BRANCH) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --tag cossacklabs/$(1):$(GIT_HASH) \ + -f ./docker/$(1).dockerfile \ + . + for tag in $(2); do \ + docker tag cossacklabs/$(1):$(GIT_HASH) cossacklabs/$(1):$$tag; \ + done +endef + +ifeq ($(BRANCH),stable) + CONTAINER_TAGS = stable latest $(VERSION) +else ifeq ($(BRANCH),master) + CONTAINER_TAGS = master current $(VERSION) +endif + +docker: + $(call docker_build,acra-build,) + $(call docker_build,acraserver,$(CONTAINER_TAGS)) + $(call docker_build,acraproxy,$(CONTAINER_TAGS)) + $(call docker_build,acra_genkeys,$(CONTAINER_TAGS)) + $(call docker_build,acra_configui,$(CONTAINER_TAGS)) + $(call docker_build,acra_genauth,$(CONTAINER_TAGS)) + @docker image rm cossacklabs/acra-build:$(GIT_HASH) + +docker_push: docker + @docker push cossacklabs/acraserver + @docker push cossacklabs/acraproxy + @docker push cossacklabs/acra_genkeys + @docker push cossacklabs/acra_genkeys + @docker push cossacklabs/acra_configui + @docker push cossacklabs/acra_genauth diff --git a/README.md b/README.md index f7fed1f5b..0adf0803e 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,15 @@ ----- -[![CircleCI](https://circleci.com/gh/cossacklabs/acra/tree/master.svg?style=shield)](https://circleci.com/gh/cossacklabs/acra) -[![Go Report Card](https://goreportcard.com/badge/github.com/cossacklabs/acra)](https://goreportcard.com/report/github.com/cossacklabs/acra) +

+ GitHub release + Circle CI + Coverage Status + +
Server platforms + Client platforms +

+
|[Documentation](https://github.com/cossacklabs/acra/wiki) | [Python sample project](https://github.com/cossacklabs/djangoproject.com) | [Ruby sample project](https://github.com/cossacklabs/rubygems.org) | [Examples](https://github.com/cossacklabs/acra/tree/master/examples) | @@ -31,7 +38,7 @@ Acra was built with specific user experiences in mind: - **automation-friendly**: most of Acra's features were built to be easily configured / automated from configuration automation environment. - **limited attack surface**: to compromise Acra-powered app, an attacker will need to compromise the separate compartmented server, AcraServer - more specifically - it's key storage, and the database. -Acra is still a product on a very early development stage: any security tools require enourmous human efforts for validation of the methods, code, and finding possible infrastructural weaknesses. Although we do run Acra in production in several instances, we're continuously enhancing it as we go to everyone's benefit. And Acra still needs ruthless dissection of all of its properties to ensure that the provided security benefits are not rendered useless through implementation problems or increased complexity. +Acra is still a product in a early development stage. And any security tools require enourmous human efforts for validation of the methods, code, and finding possible infrastructural weaknesses. Although we do run Acra in production in several instances, we're continuously enhancing and improving it as we go. And Acra still needs ruthless dissection of all of its properties to ensure that the provided security benefits are not rendered useless through implementation problems or increased complexity. ## Cryptography @@ -39,7 +46,7 @@ Acra relies on our cryptographic library [Themis](https://www.github.com/cossack ## Availability -* Acra source builds and tests with Go versions 1.6 – 1.10. +* Acra source builds and tests with Go versions 1.7 – 1.10. * Acra is known to build on: | Distributive | Go versions | @@ -47,19 +54,18 @@ Acra relies on our cryptographic library [Themis](https://www.github.com/cossack | CentOS | 1.8.3 (system) | | Debian Stretch | 1.7.4 (system) | | Debian Jessie | latest (1.3.3 is not supported) | -| Debian Wheezy | latest (1.0.2 is not supported) | | Ubuntu Artful | 1.8.3 (system) | | Ubuntu Xenial | 1.6.2 (system) | | Ubuntu Trusty | latest (1.2.1 is not supported) | | i386/Debian Stretch | 1.7.4 (system) | | i386/Debian Jessie | latest (1.3.3 is not supported) | -| i386/Debian Wheezy | latest (1.0.2 is not supported) | | i386/Ubuntu Artful | 1.8.3 (system) | | i386/Ubuntu Xenial | 1.6.2 (system) | | i386/Ubuntu Trusty | latest (1.2.1 is not supported) | -* Acra currently supports PostgreSQL 9.4+ as the database backend; MongoDB and MariaDB (and other MySQL flavours) coming quite soon. -* Acra has writer libraries for Ruby, Python, Go, and PHP, but you can easily [generate AcraStruct containers](https://github.com/cossacklabs/acra/wiki/AcraStruct) with [Themis](https://github.com/cossacklabs/themis) for any platform you want. +* Acra currently supports PostgreSQL 9.4+ as the database backend. +* Starting with Acra [`0.77.0`](https://github.com/cossacklabs/acra/releases/tag/0.77.0), we have integrated Acra with MySQL 5.7+ database, but it is still a fresh feature, which we are extensively testing to ensure its full support. Please report any MySQL bugs you may encounter through [Issues](https://github.com/cossacklabs/acra/issues). MongoDB support is coming soon, too. +* Acra has [writer libraries](https://github.com/cossacklabs/acra/wiki/Acrawriter-installation) for Ruby, Python, Go, and PHP, but you can easily [generate AcraStruct containers](https://github.com/cossacklabs/acra/wiki/AcraStruct) with [Themis](https://github.com/cossacklabs/themis) for any platform you want. ## How does Acra work? @@ -67,9 +73,10 @@ Acra relies on our cryptographic library [Themis](https://www.github.com/cossack After successfully deploying and integrating Acra into your application, follow the 4 steps below: -* Your app talks to **AcraProxy**, local daemon, via PostgreSQL driver. **AcraProxy** emulates your normal PostgreSQL database, forwards all the requests to **AcraServer** over a secure channel, and expects back plaintext output. Then **AcraProxy** forwards it over the initial PostgreSQL connection to the application. It is connected to **AcraServer** via [Secure Session](https://github.com/cossacklabs/themis/wiki/Secure-Session-cryptosystem), which ensures that all the plaintext goes over a protected channel. It is highly desirable to run **AcraProxy** via a separate user to compartmentalise it from the client-facing code. -* **AcraServer** is the core entity that provides decryption services for all the encrypted envelopes that come from the database, and then re-packs database answers for the application. -* To write protected data to the database, you can use **AcraWriter library**, which generates AcraStructs and helps you integrate it as a type into your ORM or database management code. You will need Acra's public key to do that. AcraStructs generated by AcraWriter are not readable by it - only the server has the right keys to decrypt it. +* Your app talks to **AcraProxy**, local daemon, via PostgreSQL/MySQL driver. **AcraProxy** emulates your normal PostgreSQL/MySQL database, forwards all the requests to **AcraServer** over a secure channel, and expects a plaintext output back. +* Then **AcraProxy** forwards it over the initial database connection to the application. It is connected to **AcraServer** via [Secure Session](https://github.com/cossacklabs/themis/wiki/Secure-Session-cryptosystem) or TLS, which ensures that the plaintext goes over a protected channel. It is highly desirable to run **AcraProxy** via a separate user to compartmentalise it from the client-facing code. +* **AcraServer** is the core entity that provides decryption services for all the encrypted envelopes that come from the database, and then re-packs database answers for the application. **AcraCensor** is part of AcraServer that allows customising the firewall rules for all the requests coming to the MySQL database. +* To write the protected data to the database, you can use **AcraWriter library**, which generates AcraStructs and helps you integrate it as a type into your ORM or database management code. You will need Acra's public key to do that. AcraStructs generated by AcraWriter are not readable by it — only the server has the right keys to decrypt it. * You can connect to both **AcraProxy** and the database directly when you don't need encrypted reads/writes. However, increased performance might cost you some design elegance (which is sometimes perfectly fine when it's a conscious decision). To better understand the architecture and data flow, please refer to [Architecture and data flow](https://github.com/cossacklabs/acra/wiki/Architecture-and-data-flow) section in the official documentation. @@ -77,14 +84,15 @@ To better understand the architecture and data flow, please refer to [Architectu The typical workflow looks like this: - The app encrypts some data using AcraWriter, generating AcraStruct with AcraServer public key, and updates the database. -- The app sends SQL request through AcraProxy, which forwards it to AcraServer, AcraServer forwards it to the database. +- The app sends SQL request through AcraProxy, which forwards it to AcraServer. +- AcraServer passes each query through AcraCensor, which can be configured to blacklist or whitelist some queries. AcraServer forwards the allowed queries to the database. AcraCensor can currently be only enabled for MySQL databases. - Upon receiving the answer, AcraServer tries to detect encrypted envelopes (AcraStructs). If it succeeds, AcraServer decrypts payload and replaces them with plaintext answer, which is then returned to AcraProxy over a secure channel. - AcraProxy then provides an answer to the application, as if no complex security instrumentation was ever present within the system. ## 4 steps to start -* Read the Wiki page on [building and installing](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) all the components. Soon they'll be available as pre-built binaries, but for the time being you'll need to fire a few commands to get the binaries going. -* [Deploy AcraServer](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) binaries in a separate virtual machine (or [try it in a docker container](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker)). Generate keys, put AcraServer public key into both clients (AcraProxy and AcraWriter, see next). +* Read the [Quick start guide](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) to launch all the components. We provide different ways of installing Acra: using Docker, downloading binaries, building from source. +* [Deploy AcraServer](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) binaries in a separate virtual machine (or [try it in a docker container](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker)). [Generate keys](https://github.com/cossacklabs/acra/wiki/Key-Management), put AcraServer public key into both clients (AcraProxy and AcraWriter, see next). * Deploy [AcraProxy](https://github.com/cossacklabs/acra/wiki/AcraProxy-and-AcraWriter#acraproxy) on each server where you need to read sensitive data. Generate proxy keys, provide a public key to AcraServer. Point your database access code to AcraProxy, access it as your normal database installation. * Integrate [AcraWriter](https://github.com/cossacklabs/acra/wiki/AcraProxy-and-AcraWriter#acrawriter) into your code where you need to store sensitive data, supply AcraWriter with proper server key. @@ -94,12 +102,13 @@ We fill the [Wiki documentation](https://github.com/cossacklabs/acra/wiki) with You might want to: -- Read the notes on [security design](https://github.com/cossacklabs/acra/wiki/Security-design) to better understand what you get with using Acra and what is the threat model that Acra operates in. +- Read about using the lightweight [HTTP web server AcraConfigUI](https://github.com/cossacklabs/acra/wiki/AcraConfigUI) we provide to manage AcraServer configuration in a simple fashion. +- Read the notes on [security design](https://github.com/cossacklabs/acra/wiki/Security-design) and [intrusion detection](https://github.com/cossacklabs/acra/wiki/Intrusion-detection) to better understand what you get when using Acra and what is the threat model that Acra operates in. - Read [some notes on making Acra stronger / more productive and efficient](https://github.com/cossacklabs/acra/wiki/Tuning-Acra), and on adding security features or increasing throughput, depending on your goals and security model. - +- Read about the [logging format](https://github.com/cossacklabs/acra/wiki/Logging) that Acra supports if you are using any SIEM system. ## Project status -This open source version of Acra is an early alpha. We're slowly unifying and moving features from its previous incarnation into a community-friendly edition. Please let us know in the [Issues](https://www.github.com/cossacklabs/acra/issues) whenever you stumble upon a bug, see a possible enhancement or have a comment on security design. +This open source version of Acra is an early beta. We're slowly unifying and moving features from its previous incarnation into a community-friendly edition. Please let us know in the [Issues](https://www.github.com/cossacklabs/acra/issues) whenever you stumble upon a bug, see a possible enhancement, or have a comment on security design. ## Contributing to us If you’d like to contribute your code or other kind of input to Acra, you’re very welcome. Your starting point for contributing should be this [Contribution Wiki page](https://github.com/cossacklabs/acra/wiki/Contributing-to-Acra). diff --git a/acracensor/acracensor_configuration_provider.go b/acracensor/acracensor_configuration_provider.go new file mode 100644 index 000000000..5a08cc3b4 --- /dev/null +++ b/acracensor/acracensor_configuration_provider.go @@ -0,0 +1,80 @@ +package acracensor + +import ( + "github.com/cossacklabs/acra/acracensor/handlers" + "gopkg.in/yaml.v2" + "strings" +) + +const BlacklistConfigStr = "blacklist" +const WhitelistConfigStr = "whitelist" +const LoggerConfigStr = "logger" + +type AcracensorConfig struct { + Handlers []struct { + Handler string + Queries []string + Tables []string + Rules []string + Filepath string + } +} + +func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) error { + err := acraCensor.update(configuration) + if err != nil { + return err + } + return nil +} + +func (acraCensor *AcraCensor) update(configuration []byte) error { + var censorConfiguration AcracensorConfig + err := yaml.Unmarshal(configuration, &censorConfiguration) + if err != nil { + return err + } + for _, handlerConfiguration := range censorConfiguration.Handlers { + switch handlerConfiguration.Handler { + case WhitelistConfigStr: + whitelistHandler := &handlers.WhitelistHandler{} + err := whitelistHandler.AddQueries(handlerConfiguration.Queries) + if err != nil { + return err + } + whitelistHandler.AddTables(handlerConfiguration.Tables) + err = whitelistHandler.AddRules(handlerConfiguration.Rules) + if err != nil { + return err + } + acraCensor.AddHandler(whitelistHandler) + break + case BlacklistConfigStr: + blacklistHandler := &handlers.BlacklistHandler{} + err := blacklistHandler.AddQueries(handlerConfiguration.Queries) + if err != nil { + return err + } + blacklistHandler.AddTables(handlerConfiguration.Tables) + err = blacklistHandler.AddRules(handlerConfiguration.Rules) + if err != nil { + return err + } + acraCensor.AddHandler(blacklistHandler) + break + case LoggerConfigStr: + if strings.EqualFold(handlerConfiguration.Filepath, ""){ + break + } + logger, err := handlers.NewLoggingHandler(handlerConfiguration.Filepath) + if err != nil { + return err + } + acraCensor.AddHandler(logger) + break + default: + break + } + } + return nil +} diff --git a/acracensor/acracensor_implementation.go b/acracensor/acracensor_implementation.go new file mode 100644 index 000000000..997452082 --- /dev/null +++ b/acracensor/acracensor_implementation.go @@ -0,0 +1,33 @@ +package acracensor + +import ( + log "github.com/sirupsen/logrus" +) + +type AcraCensor struct { + handlers []QueryHandlerInterface + +} + +func (acraCensor *AcraCensor) AddHandler(handler QueryHandlerInterface) { + acraCensor.handlers = append(acraCensor.handlers, handler) +} + +func (acraCensor *AcraCensor) RemoveHandler(handler QueryHandlerInterface) { + for index, handlerFromRange := range acraCensor.handlers { + if handlerFromRange == handler { + acraCensor.handlers = append(acraCensor.handlers[:index], acraCensor.handlers[index+1:]...) + } + } +} + +func (acraCensor *AcraCensor) HandleQuery(query string) error { + for _, handler := range acraCensor.handlers { + if err := handler.CheckQuery(query); err != nil { + log.Errorf("Forbidden query: '%s'", query) + return err + } + } + log.Infof("Allowed query: '%s'", query) + return nil +} diff --git a/firewall/firewall_interfaces.go b/acracensor/acracensor_interfaces.go similarity index 71% rename from firewall/firewall_interfaces.go rename to acracensor/acracensor_interfaces.go index 15b0fbca0..1d56f86ca 100644 --- a/firewall/firewall_interfaces.go +++ b/acracensor/acracensor_interfaces.go @@ -1,16 +1,13 @@ -package firewall - +package acracensor type QueryHandlerInterface interface { CheckQuery(sqlQuery string) error + Reset() + GetName() string } - -type FirewallInterface interface { +type AcracensorInterface interface { HandleQuery(sqlQuery string) error AddHandler(handler QueryHandlerInterface) RemoveHandler(handler QueryHandlerInterface) } - - - diff --git a/acracensor/acracensor_test.go b/acracensor/acracensor_test.go new file mode 100644 index 000000000..d7f9eba7d --- /dev/null +++ b/acracensor/acracensor_test.go @@ -0,0 +1,734 @@ +package acracensor + +import ( + "github.com/cossacklabs/acra/acracensor/handlers" + "github.com/cossacklabs/acra/utils" + "io/ioutil" + "os" + "path/filepath" + "testing" + "strings" +) + +func TestWhitelistQueries(t *testing.T) { + + sqlSelectQueries := []string{ + "SELECT * FROM Schema.Tables;", + "SELECT Student_ID FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", + "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", + "SELECT SUM(Salary)FROM Employee WHERE Emp_Age < 30;", + "SELECT AVG(Price)FROM Products;", + } + + sqlInsertQueries := []string{ + "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", + "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Y');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Z');", + "INSERT INTO SalesStaff3 (StaffID, FullNameTbl) VALUES (X, M);", + "INSERT INTO X.Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "INSERT INTO Production (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO T1 (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO dbo.Points (Type, PointValue) VALUES ('Point', '1,5');", + "INSERT INTO dbo.Points (PointValue) VALUES ('1,99');", + } + + whitelistHandler := &handlers.WhitelistHandler{} + + err := whitelistHandler.AddQueries(sqlSelectQueries) + if err != nil { + t.Fatal(err) + } + err = whitelistHandler.AddQueries(sqlInsertQueries) + if err != nil { + t.Fatal(err) + } + + acraCensor := &AcraCensor{} + + //set our acracensor to use whitelist for query evaluating + acraCensor.AddHandler(whitelistHandler) + + //acracensor should not block those queries + for _, query := range sqlSelectQueries { + err = acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + for _, query := range sqlInsertQueries { + err = acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + //acracensor should block this query because it is not in whitelist + err = acraCensor.HandleQuery("SELECT * FROM Schema.views;") + if err != handlers.ErrQueryNotInWhitelist { + t.Fatal(err) + } + + //ditto + err = acraCensor.HandleQuery("INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');") + if err != handlers.ErrQueryNotInWhitelist { + t.Fatal(err) + } + + testWhitelistTables(t, acraCensor, whitelistHandler) + testWhitelistRules(t, acraCensor, whitelistHandler) +} +func testWhitelistTables(t *testing.T, acraCensor *AcraCensor, whitelistHandler *handlers.WhitelistHandler) { + + testQueries := []string{ + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL AS EMPL_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + } + + err := whitelistHandler.AddQueries(testQueries) + if err != nil { + t.Fatal(err) + } + + whitelistHandler.AddTables([]string{"EMPLOYEE"}) + + queryIndexesToBlock := []int{0, 2, 3, 4, 5, 6} + + //acracensor should block those queries + for _, i := range queryIndexesToBlock { + err := acraCensor.HandleQuery(testQueries[i]) + if err != handlers.ErrAccessToForbiddenTableWhitelist { + t.Fatal(err) + } + } + + err = acraCensor.HandleQuery(testQueries[1]) + //acracensor should not block this query + if err != nil { + t.Fatal(err) + } + + //Now we have no tables in whitelist, so should block all queries + whitelistHandler.RemoveTables([]string{"EMPLOYEE"}) + + //acracensor should not block queries + for _, query := range testQueries { + err = acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + testQuery := "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL, CUSTOMERS WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;" + + err = whitelistHandler.AddQueries([]string{testQuery}) + if err != nil { + t.Fatal(err) + } + whitelistHandler.AddTables([]string{"EMPLOYEE", "EMPLOYEE_TBL"}) + + err = acraCensor.HandleQuery(testQuery) + //acracensor should block this query + if err != handlers.ErrAccessToForbiddenTableWhitelist { + t.Fatal(err) + } + + whitelistHandler.AddTables([]string{"CUSTOMERS"}) + + err = acraCensor.HandleQuery(testQuery) + + //acracensor should not block this query + if err != nil { + t.Fatal(err) + } +} +func testWhitelistRules(t *testing.T, acraCensor *AcraCensor, whitelistHandler *handlers.WhitelistHandler) { + whitelistHandler.Reset() + + testQueries := []string{ + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL AS EMPL_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + } + + //acracensor should block all queries except accessing to any information but only in table EMPLOYEE_TBL and related only to Seattle city [1,2,3] + testSecurityRules := []string{ + "SELECT * FROM EMPLOYEE_TBL WHERE CITY='Seattle'", + } + + queryIndexesToBlock := []int{1, 2, 3, 5} + err := whitelistHandler.AddRules(testSecurityRules) + if err != nil { + t.Fatal(err) + } + + //acracensor should block those queries + for _, i := range queryIndexesToBlock { + err := acraCensor.HandleQuery(testQueries[i]) + if err != handlers.ErrForbiddenSqlStructureWhitelist { + t.Fatal(err) + } + } + + queryIndexesToPass := []int{0, 4} + //acracensor should not block those queries + for _, i := range queryIndexesToPass { + err := acraCensor.HandleQuery(testQueries[i]) + if err != nil { + t.Fatal(err) + } + } + + whitelistHandler.RemoveRules(testSecurityRules) + //acracensor should not block all queries + for _, query := range testQueries { + err := acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } +} + +func TestBlacklistQueries(t *testing.T) { + sqlSelectQueries := []string{ + "SELECT * FROM Schema.Tables;", + "SELECT * FROM Schema.Tables;", + "SELECT * FROM Schema.Tables;", + "SELECT Student_ID FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", + "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", + "SELECT SUM(Salary) FROM Employee WHERE Emp_Age < 30;", + "SELECT * FROM Schema.views;", + } + + sqlInsertQueries := []string{ + "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", + "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", + "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", + "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", + "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "INSERT INTO films VALUES ('UA502', 'Bananas', 105, '1971-07-13', 'Comedy', '82 minutes');", + "INSERT INTO films (code, title, did, date_prod, kind) VALUES ('B6717', 'Tampopo', 110, '1985-02-10', 'Comedy'), ('HG120', 'The Dinner Game', 140, DEFAULT, 'Comedy');", + "INSERT INTO films SELECT * FROM tmp_films WHERE date_prod < '2004-05-07';", + } + + blackList := []string{ + "INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');", + "SELECT AVG(Price) FROM Products;", + } + + blacklistHandler := &handlers.BlacklistHandler{} + err := blacklistHandler.AddQueries(blackList) + if err != nil { + t.Fatal(err) + } + + acraCensor := &AcraCensor{} + + //set our acracensor to use blacklist for query evaluating + acraCensor.AddHandler(blacklistHandler) + + //acracensor should not block those queries + for _, query := range sqlSelectQueries { + err = acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + for _, query := range sqlInsertQueries { + err = acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + testQuery := "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');" + + err = blacklistHandler.AddQueries([]string{testQuery}) + if err != nil { + t.Fatal(err) + } + + err = acraCensor.HandleQuery(testQuery) + //acracensor should block this query because it's in blacklist + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + + acraCensor.RemoveHandler(blacklistHandler) + + err = acraCensor.HandleQuery(testQuery) + //acracensor should not block this query because we removed blacklist handler, err should be nil + if err != nil { + t.Fatal(err) + } + + //again set our acracensor to use blacklist for query evaluating + acraCensor.AddHandler(blacklistHandler) + err = acraCensor.HandleQuery(testQuery) + + //now acracensor should block testQuery because it's in blacklist + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + + blacklistHandler.RemoveQueries([]string{testQuery}) + + err = acraCensor.HandleQuery(testQuery) + //now acracensor should not block testQuery + if err != nil { + t.Fatal(err) + } + + testBlacklistTables(t, acraCensor, blacklistHandler) + + testBlacklistRules(t, acraCensor, blacklistHandler) +} +func testBlacklistTables(t *testing.T, censor *AcraCensor, blacklistHandler *handlers.BlacklistHandler) { + + blacklistHandler.Reset() + + testQueries := []string{ + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL AS EMPL_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + } + + blacklistHandler.AddTables([]string{"EMPLOYEE_TBL", "Customers"}) + + //acracensor should block these queries + queryIndexesToBlock := []int{0, 2, 4, 5, 6} + for _, i := range queryIndexesToBlock { + err := censor.HandleQuery(testQueries[i]) + if err != handlers.ErrAccessToForbiddenTableBlacklist { + t.Fatal(err) + } + } + + //acracensor should not block these queries + queryIndexesToPass := []int{1, 3} + for _, i := range queryIndexesToPass { + err := censor.HandleQuery(testQueries[i]) + if err != nil { + t.Fatal(err) + } + } + + blacklistHandler.RemoveTables([]string{"EMPLOYEE_TBL"}) + + err := censor.HandleQuery(testQueries[0]) + //acracensor should not block this query + if err != nil { + t.Fatal(err) + } + + err = censor.HandleQuery(testQueries[2]) + //acracensor should not block this query + if err != nil { + t.Fatal(err) + } + +} +func testBlacklistRules(t *testing.T, acraCensor *AcraCensor, blacklistHandler *handlers.BlacklistHandler) { + + blacklistHandler.Reset() + + testQueries := []string{ + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE, EMPLOYEE_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL AS EMPL_TBL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + } + + //acracensor should block all queries that try to access to information in table EMPLOYEE_TBL related to Seattle city + testSecurityRules := []string{ + "SELECT * FROM EMPLOYEE_TBL WHERE CITY='Seattle'", + } + + queryIndexesToBlock := []int{0, 2, 4} + + err := blacklistHandler.AddRules(testSecurityRules) + if err != nil { + t.Fatal(err) + } + + //acracensor should block those queries + for _, i := range queryIndexesToBlock { + err := acraCensor.HandleQuery(testQueries[i]) + if err != handlers.ErrForbiddenSqlStructureBlacklist { + t.Fatal(err) + } + } + + queryIndexesToPass := []int{1, 3} + //acracensor should not block those queries + for _, i := range queryIndexesToPass { + err := acraCensor.HandleQuery(testQueries[i]) + if err != nil { + t.Fatal(err) + } + } + + blacklistHandler.RemoveRules(testSecurityRules) + //acracensor should not block all queries + for _, query := range testQueries { + err := acraCensor.HandleQuery(query) + if err != nil { + t.Fatal(err) + } + } + + testSecurityRules = []string{ + "SELECT * FROM EMPLOYEE_TBL, EMPLOYEE WHERE CITY='Seattle'", + "SELECT * FROM EMPLOYEE_TBL, EMPLOYEE WHERE CITY='INDIANAPOLIS'", + } + + blacklistHandler.Reset() + err = blacklistHandler.AddRules(testSecurityRules) + if err != nil { + t.Fatal(err) + } + //acracensor should block all queries + for _, query := range testQueries { + err := acraCensor.HandleQuery(query) + if err != handlers.ErrForbiddenSqlStructureBlacklist { + t.Fatal(err) + } + } +} + +func TestConfigurationProvider(t *testing.T) { + + var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName("acra_censor.example") + + filePath, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + configuration, err := ioutil.ReadFile(filepath.Join(filePath, "../", DEFAULT_CONFIG_PATH)) + if err != nil { + t.Fatal(err) + } + + acraCensor := &AcraCensor{} + + err = acraCensor.LoadConfiguration(configuration) + if err != nil { + t.Fatal(err) + } + + testQueries := []string{ + "INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');", + "SELECT AVG(Price) FROM Products;", + } + + //acracensor should block those queries (blacklist works) + for _, queryToBlock := range testQueries { + err = acraCensor.HandleQuery(queryToBlock) + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + } + + testQueries = []string{ + "INSERT INTO EMPLOYEE_TBL VALUES (1, 'Stephen', 'Jiang');", + "SELECT AVG(Price) FROM Customers;", + } + + //acracensor should block those tables (blacklist works) + for _, queryToBlock := range testQueries { + err = acraCensor.HandleQuery(queryToBlock) + if err != handlers.ErrAccessToForbiddenTableBlacklist { + t.Fatal(err) + } + } + + testQueries = []string{ + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE AS EMPL WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + } + + //acracensor should block those structures (blacklist works) + for _, queryToBlock := range testQueries { + err = acraCensor.HandleQuery(queryToBlock) + if err != handlers.ErrForbiddenSqlStructureBlacklist { + t.Fatal(err) + } + } + + testQueries = []string{ + "SELECT EMP_ID, LAST_NAME FROM PRODUCTS WHERE CITY='INDIANAPOLIS' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM PRODUCTS WHERE CITY='INDIANAPOLIS' ORDER BY EMP_ID asc;", + } + + //acracensor should block those tables (whitelist works) + for _, queryToBlock := range testQueries { + err = acraCensor.HandleQuery(queryToBlock) + if err != handlers.ErrAccessToForbiddenTableWhitelist { + t.Fatal(err) + } + } + + expectedQueriesInCensorLog := "[{\"RawQuery\":\"INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT AVG(Price) FROM Products;\",\"IsForbidden\":false},{\"RawQuery\":\"INSERT INTO EMPLOYEE_TBL VALUES (1, 'Stephen', 'Jiang');\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT AVG(Price) FROM Customers;\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT EMP_ID, LAST_NAME FROM EMPLOYEE AS EMPL WHERE CITY = 'Seattle' ORDER BY EMP_ID;\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT EMP_ID, LAST_NAME FROM PRODUCTS WHERE CITY='INDIANAPOLIS' ORDER BY EMP_ID;\",\"IsForbidden\":false},{\"RawQuery\":\"SELECT EMP_ID, LAST_NAME FROM PRODUCTS WHERE CITY='INDIANAPOLIS' ORDER BY EMP_ID asc;\",\"IsForbidden\":false}]" + + censorLogsBytes, err := ioutil.ReadFile("censor_log") + if err != nil { + t.Fatal(err) + } + + if !strings.EqualFold(expectedQueriesInCensorLog, string(censorLogsBytes)){ + t.Fatal("Configuration parsing logic error 1") + } + + err = os.Remove("censor_log") + if err != nil { + t.Fatal(err) + } + + testSyntax(t) +} +func testSyntax(t *testing.T) { + + acraCensor := &AcraCensor{} + + configuration := `handlers: + - handler: blacklist + queries: + - INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang'); + - SLECT AVG(Price) FROM Products;` + + err := acraCensor.LoadConfiguration([]byte(configuration)) + if err != handlers.ErrQuerySyntaxError { + t.Fatal(err) + } + + configuration = `handlers: + - handler: blacklist + queries: + - INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang'); + - SELECT AVG(Price) FROM Products; + tables: + - EMPLOYEE_TBL + - Customers + rules: + - SELECT * ROM EMPLOYEE WHERE CITY='Seattle';` + + err = acraCensor.LoadConfiguration([]byte(configuration)) + if err != handlers.ErrStructureSyntaxError { + t.Fatal(err) + } +} + +func TestSerialization(t *testing.T){ + testQueries := []string{ + "SELECT * FROM Schema.Tables;", + "SELECT Student_ID FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT * FROM X;", + "SELECT * FROM Y;", + "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", + "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", + "SELECT SUM(Salary)FROM Employee WHERE Emp_Age < 30;", + "SELECT AVG(Price)FROM Products;", + "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", + "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Y');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Z');", + "INSERT INTO SalesStaff3 (StaffID, FullNameTbl) VALUES (X, M);", + "INSERT INTO X.Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "INSERT INTO Production (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO T1 (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO dbo.Points (Type, PointValue) VALUES ('Point', '1,5');", + "INSERT INTO dbo.Points (PointValue) VALUES ('1,99');", + } + + tmpFile, err := ioutil.TempFile("", "censor_log") + if err != nil { + t.Fatal(err) + } + + if err = tmpFile.Close(); err != nil { + t.Fatal(err) + } + + loggingHandler, err := handlers.NewLoggingHandler(tmpFile.Name()) + if err != nil { + t.Fatal(err) + } + + for _, query := range testQueries { + err = loggingHandler.CheckQuery(query) + if err != nil { + t.Fatal(err) + } + } + + if len(loggingHandler.GetAllInputQueries()) != len(testQueries){ + t.Fatal("loggingHandler logic error 1") + } + + err = loggingHandler.Serialize() + if err != nil { + t.Fatal(err) + } + + loggingHandler.Reset() + + if len(loggingHandler.GetAllInputQueries()) != 0 { + t.Fatal("loggingHandler logic error 2") + } + + err = loggingHandler.Deserialize() + if err != nil { + t.Fatal(err) + } + + if len(loggingHandler.GetAllInputQueries()) != len(testQueries){ + t.Fatal("loggingHandler logic error 3") + } + + for index, query := range loggingHandler.GetAllInputQueries(){ + if testQueries[index] != query{ + t.Fatal("loggingHandler logic error 4") + } + } + + if err = os.Remove(tmpFile.Name()); err != nil { + t.Fatal(err) + } +} +func TestLogging(t *testing.T){ + + testQueries := []string{ + "SELECT * FROM Schema.Tables;", + "SELECT Student_ID FROM STUDENT;", + "SELECT * FROM STUDENT;", + "SELECT * FROM X;", + "SELECT * FROM Y;", + "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", + "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", + "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", + "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", + "SELECT SUM(Salary)FROM Employee WHERE Emp_Age < 30;", + "SELECT AVG(Price)FROM Products;", + "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", + "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Y');", + "INSERT INTO SalesStaff3 (StaffID, FullName) VALUES (X, 'Z');", + "INSERT INTO SalesStaff3 (StaffID, FullNameTbl) VALUES (X, M);", + "INSERT INTO X.Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", + "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", + "INSERT INTO Production (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO T1 (Name, UnitMeasureCode, ModifiedDate) VALUES ('Square Yards', 'Y2', GETDATE());", + "INSERT INTO dbo.Points (Type, PointValue) VALUES ('Point', '1,5');", + "INSERT INTO dbo.Points (PointValue) VALUES ('1,99');", + } + + tmpFile, err := ioutil.TempFile("", "censor_log") + if err != nil { + t.Fatal(err) + } + + if err = tmpFile.Close(); err != nil { + t.Fatal(err) + } + + loggingHandler, err := handlers.NewLoggingHandler(tmpFile.Name()) + if err != nil { + t.Fatal(err) + } + + blacklist := &handlers.BlacklistHandler{} + + acraCensor := &AcraCensor{} + acraCensor.AddHandler(loggingHandler) + acraCensor.AddHandler(blacklist) + + for _, testQuery := range testQueries { + err = acraCensor.HandleQuery(testQuery) + if err != nil { + t.Fatal(err) + } + } + + err = loggingHandler.MarkQueryAsForbidden(testQueries[0]) + if err != nil { + t.Fatal(err) + } + err = loggingHandler.MarkQueryAsForbidden(testQueries[1]) + if err != nil { + t.Fatal(err) + } + err = loggingHandler.MarkQueryAsForbidden(testQueries[2]) + if err != nil { + t.Fatal(err) + } + + err = blacklist.AddQueries(loggingHandler.GetForbiddenQueries()) + if err != nil { + t.Fatal(err) + } + + err = acraCensor.HandleQuery(testQueries[0]) + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + + err = acraCensor.HandleQuery(testQueries[1]) + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + + err = acraCensor.HandleQuery(testQueries[2]) + if err != handlers.ErrQueryInBlacklist { + t.Fatal(err) + } + + //zero, first and second query are forbidden + for index := 3; index < len(testQueries); index++ { + err = acraCensor.HandleQuery(testQueries[index]) + if err != nil { + t.Fatal(err) + } + } + + if err = os.Remove(tmpFile.Name()); err != nil { + t.Fatal(err) + } +} diff --git a/acracensor/handlers/blacklist_handler.go b/acracensor/handlers/blacklist_handler.go new file mode 100644 index 000000000..8e4d62d62 --- /dev/null +++ b/acracensor/handlers/blacklist_handler.go @@ -0,0 +1,246 @@ +package handlers + +import ( + "errors" + "github.com/xwb1989/sqlparser" + "reflect" + "strings" +) + +type BlacklistHandler struct { + queries []string + tables []string + rules []string +} + +func (handler *BlacklistHandler) CheckQuery(query string) error { + + //Check queries + if len(handler.queries) != 0 { + //Check that query is not in blacklist + yes, _ := contains(handler.queries, query) + if yes { + return ErrQueryInBlacklist + } + } + + //Check tables + if len(handler.tables) != 0 { + parsedQuery, err := sqlparser.Parse(query) + if err != nil { + return ErrParseTablesBlacklist + } + + switch parsedQuery := parsedQuery.(type) { + case *sqlparser.Select: + for _, forbiddenTable := range handler.tables { + for _, table := range parsedQuery.From { + if strings.EqualFold(sqlparser.String(table.(*sqlparser.AliasedTableExpr).Expr), forbiddenTable) { + return ErrAccessToForbiddenTableBlacklist + } + } + } + + case *sqlparser.Insert: + for _, forbiddenTable := range handler.tables { + if strings.EqualFold(parsedQuery.Table.Name.String(), forbiddenTable) { + return ErrAccessToForbiddenTableBlacklist + } + } + + case *sqlparser.Update: + return ErrNotImplemented + } + } + + //Check rules + if len(handler.rules) != 0 { + violationOccured, err := handler.testRulesViolation(query) + if err != nil { + return ErrParseSqlRuleBlacklist + } + if violationOccured { + return ErrForbiddenSqlStructureBlacklist + } + } + return nil +} + +func (handler *BlacklistHandler) Reset() { + handler.queries = nil + handler.tables = nil + handler.rules = nil +} + +func (handler *BlacklistHandler) GetName() string { + return "Blacklist" +} + +func (handler *BlacklistHandler) AddQueries(queries []string) error { + + for _, query := range queries { + _, err := sqlparser.Parse(query) + if err != nil { + return ErrQuerySyntaxError + } + handler.queries = append(handler.queries, query) + } + + handler.queries = removeDuplicates(handler.queries) + + return nil +} + +func (handler *BlacklistHandler) RemoveQueries(queries []string) { + + for _, query := range queries { + yes, index := contains(handler.queries, query) + if yes { + handler.queries = append(handler.queries[:index], handler.queries[index+1:]...) + } + } +} + +func (handler *BlacklistHandler) AddTables(tableNames []string) { + + for _, tableName := range tableNames { + handler.tables = append(handler.tables, tableName) + } + + handler.tables = removeDuplicates(handler.tables) +} + +func (handler *BlacklistHandler) RemoveTables(tableNames []string) { + + for _, query := range tableNames { + yes, index := contains(handler.tables, query) + if yes { + handler.tables = append(handler.tables[:index], handler.tables[index+1:]...) + } + } +} + +func (handler *BlacklistHandler) AddRules(rules []string) error { + for _, rule := range rules { + handler.rules = append(handler.rules, rule) + _, err := sqlparser.Parse(rule) + if err != nil { + return ErrStructureSyntaxError + } + } + + handler.rules = removeDuplicates(handler.rules) + + return nil +} + +func (handler *BlacklistHandler) RemoveRules(rules []string) { + for _, rule := range rules { + yes, index := contains(handler.rules, rule) + if yes { + handler.rules = append(handler.rules[:index], handler.rules[index+1:]...) + } + } +} + +func (handler *BlacklistHandler) testRulesViolation(query string) (bool, error) { + + if sqlparser.Preview(query) != sqlparser.StmtSelect { + return true, errors.New("non-select queries are not supported") + } + + //parse one rule and get forbidden tables and columns for specific 'where' clause + var whereClause sqlparser.SQLNode + var tables sqlparser.TableExprs + var columns sqlparser.SelectExprs + + //Parse each rule and then test query + for _, rule := range handler.rules { + parsedRule, err := sqlparser.Parse(rule) + if err != nil { + return true, err + } + + switch parsedRule := parsedRule.(type) { + + case *sqlparser.Select: + whereClause = parsedRule.Where.Expr + tables = parsedRule.From + columns = parsedRule.SelectExprs + + dangerousSelect, err := handler.isDangerousSelect(query, whereClause, tables, columns) + if err != nil { + return true, err + } + + if dangerousSelect { + return true, nil + } + + case *sqlparser.Insert: + return true, ErrNotImplemented + default: + return true, ErrNotImplemented + } + + _ = whereClause + _ = tables + _ = columns + } + + return false, nil +} + +func (handler *BlacklistHandler) isDangerousSelect(selectQuery string, forbiddenWhere sqlparser.SQLNode, forbiddenTables sqlparser.TableExprs, forbiddenColumns sqlparser.SelectExprs) (bool, error) { + + parsedSelectQuery, err := sqlparser.Parse(selectQuery) + if err != nil { + return true, err + } + + evaluatedStmt := parsedSelectQuery.(*sqlparser.Select) + + if evaluatedStmt.Where != nil { + if strings.EqualFold(sqlparser.String(forbiddenWhere), sqlparser.String(evaluatedStmt.Where.Expr)) { + if handler.isForbiddenTableAccess(evaluatedStmt.From, forbiddenTables) { + if handler.isForbiddenColumnAccess(evaluatedStmt.SelectExprs, forbiddenColumns) { + return true, nil + } + } + } + } else { + if handler.isForbiddenTableAccess(evaluatedStmt.From, forbiddenTables) { + if handler.isForbiddenColumnAccess(evaluatedStmt.SelectExprs, forbiddenColumns) { + return true, nil + } + } + } + + return false, nil +} + +func (handler *BlacklistHandler) isForbiddenTableAccess(tablesToEvaluate sqlparser.TableExprs, forbiddenTables sqlparser.TableExprs) bool { + for _, tableToEvaluate := range tablesToEvaluate { + for _, forbiddenTable := range forbiddenTables { + if reflect.DeepEqual(tableToEvaluate.(*sqlparser.AliasedTableExpr).Expr, forbiddenTable.(*sqlparser.AliasedTableExpr).Expr) { + return true + } + } + } + return false +} + +func (handler *BlacklistHandler) isForbiddenColumnAccess(columnsToEvaluate sqlparser.SelectExprs, forbiddenColumns sqlparser.SelectExprs) bool { + if strings.EqualFold(sqlparser.String(forbiddenColumns), "*") { + return true + } + + for _, columnToEvaluate := range columnsToEvaluate { + for _, forbiddenColumn := range forbiddenColumns { + if reflect.DeepEqual(columnToEvaluate, forbiddenColumn) { + return true + } + } + } + return false +} diff --git a/acracensor/handlers/handlers_util.go b/acracensor/handlers/handlers_util.go new file mode 100644 index 000000000..af2864d4d --- /dev/null +++ b/acracensor/handlers/handlers_util.go @@ -0,0 +1,50 @@ +package handlers + +import ( + "errors" + "strings" +) + +var ErrQueryNotInWhitelist = errors.New("query not in whitelist") +var ErrQueryInBlacklist = errors.New("query in blacklist") + +var ErrAccessToForbiddenTableBlacklist = errors.New("query tries to access forbidden table") +var ErrAccessToForbiddenTableWhitelist = errors.New("query tries to access forbidden table") + +var ErrForbiddenSqlStructureBlacklist = errors.New("query's structure is forbidden") +var ErrForbiddenSqlStructureWhitelist = errors.New("query's structure is forbidden") + +var ErrParseTablesBlacklist = errors.New("parsing tables error") +var ErrParseSqlRuleBlacklist = errors.New("parsing security rules error") + +var ErrParseTablesWhitelist = errors.New("parsing tables error") +var ErrParseSqlRuleWhitelist = errors.New("parsing security rules error") + +var ErrNotImplemented = errors.New("not implemented yet") + +var ErrQuerySyntaxError = errors.New("fail to parse specified query") +var ErrStructureSyntaxError = errors.New("fail to parse specified structure") + +func removeDuplicates(input []string) []string { + + keys := make(map[string]bool) + var result []string + for _, entry := range input { + if _, value := keys[entry]; !value { + keys[entry] = true + result = append(result, entry) + } + } + return result +} + +func contains(queries []string, query string) (bool, int) { + + for index, queryFromRange := range queries { + if strings.EqualFold(queryFromRange, query) { + + return true, index + } + } + return false, 0 +} diff --git a/acracensor/handlers/handlers_util_test.go b/acracensor/handlers/handlers_util_test.go new file mode 100644 index 000000000..56c911d1a --- /dev/null +++ b/acracensor/handlers/handlers_util_test.go @@ -0,0 +1,46 @@ +package handlers + +import ( + "strings" + "testing" +) + +func TestUtilities(t *testing.T) { + + //Test 1 + expected := []string{"x", "y", "z"} + + input := []string{"x", "y", "z", "x", "y"} + + output := removeDuplicates(input) + + if !areEqual(output, expected) { + t.Fatal("unexpected result") + } + + //Test 2 + expected = []string{"@lagovas", "@vixentael", "@secumod"} + + input = []string{"@lagovas", "@vixentael", "@secumod", "@lagovas", "@vixentael", "@secumod", "@lagovas", "@vixentael", "@secumod"} + + output = removeDuplicates(input) + + if !areEqual(output, expected) { + t.Fatal("unexpected result") + } + +} + +func areEqual(a []string, b []string) bool { + if len(a) != len(b) { + return false + } + + for index := 0; index < len(a); index++ { + if !strings.EqualFold(a[index], b[index]) { + return false + } + } + + return true +} diff --git a/acracensor/handlers/logging_handler.go b/acracensor/handlers/logging_handler.go new file mode 100644 index 000000000..ac3ee2b3f --- /dev/null +++ b/acracensor/handlers/logging_handler.go @@ -0,0 +1,97 @@ +package handlers + +import ( + "strings" + "io/ioutil" + "encoding/json" + "os" +) + +type LoggingHandler struct { + Queries []QueryInfo + filePath string +} + +type QueryInfo struct { + RawQuery string + IsForbidden bool +} + +func NewLoggingHandler (filePath string) (*LoggingHandler, error) { + file, err := os.OpenFile(filePath, os.O_RDONLY|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + err = file.Close() + if err != nil { + return nil, err + } + return &LoggingHandler{Queries:nil, filePath:filePath}, nil +} + +func (handler *LoggingHandler) CheckQuery(query string) error { + //skip already logged queries + for _, queryInfo := range handler.Queries{ + if strings.EqualFold(queryInfo.RawQuery, query){ + return nil + } + } + queryInfo := &QueryInfo{} + queryInfo.RawQuery = query + queryInfo.IsForbidden = false + handler.Queries = append(handler.Queries, *queryInfo) + return handler.Serialize() +} + +func (handler *LoggingHandler) Reset() { + handler.Queries = nil +} + +func (handler *LoggingHandler) GetName() string{ + return "Logging" +} + +func (handler *LoggingHandler) GetAllInputQueries() []string{ + var queries []string + for _, queryInfo := range handler.Queries { + queries = append(queries, queryInfo.RawQuery) + } + return queries +} + +func (handler *LoggingHandler) MarkQueryAsForbidden(query string) error { + for index, queryInfo := range handler.Queries { + if strings.EqualFold(query, queryInfo.RawQuery) { + handler.Queries[index].IsForbidden = true + } + } + return handler.Serialize() +} + +func (handler *LoggingHandler) GetForbiddenQueries() []string{ + var forbiddenQueries []string + for _, queryInfo := range handler.Queries { + if queryInfo.IsForbidden == true{ + forbiddenQueries = append(forbiddenQueries, queryInfo.RawQuery) + } + } + return forbiddenQueries +} + +func (handler *LoggingHandler) Serialize() error { + jsonFile, err := json.Marshal(handler.Queries) + if err != nil { + return err + } + return ioutil.WriteFile(handler.filePath, jsonFile, 0600) + +} + +func (handler *LoggingHandler) Deserialize() error { + var bufferBytes []byte + bufferBytes, err := ioutil.ReadFile(handler.filePath) + if err != nil { + return err + } + return json.Unmarshal(bufferBytes, &handler.Queries) +} \ No newline at end of file diff --git a/acracensor/handlers/whitelist_handler.go b/acracensor/handlers/whitelist_handler.go new file mode 100644 index 000000000..1a57b8313 --- /dev/null +++ b/acracensor/handlers/whitelist_handler.go @@ -0,0 +1,222 @@ +package handlers + +import ( + "errors" + "github.com/xwb1989/sqlparser" + "reflect" + "strings" +) + +type WhitelistHandler struct { + queries []string + tables []string + rules []string +} + +func (handler *WhitelistHandler) CheckQuery(query string) error { + + //Check queries + if len(handler.queries) != 0 { + yes, _ := contains(handler.queries, query) + if !yes { + return ErrQueryNotInWhitelist + } + } + + //Check tables + if len(handler.tables) != 0 { + parsedQuery, err := sqlparser.Parse(query) + if err != nil { + return ErrParseTablesWhitelist + } + + switch parsedQuery := parsedQuery.(type) { + case *sqlparser.Select: + allowedTablesCounter := 0 + for _, allowedTable := range handler.tables { + for _, table := range parsedQuery.From { + if strings.EqualFold(sqlparser.String(table.(*sqlparser.AliasedTableExpr).Expr), allowedTable) { + allowedTablesCounter++ + } + } + } + if allowedTablesCounter != len(parsedQuery.From) { + return ErrAccessToForbiddenTableWhitelist + } + + case *sqlparser.Insert: + + tableIsAllowed := false + for _, allowedTable := range handler.tables { + if strings.EqualFold(parsedQuery.Table.Name.String(), allowedTable) { + tableIsAllowed = true + } + } + if !tableIsAllowed { + return ErrAccessToForbiddenTableWhitelist + } + case *sqlparser.Update: + } + } + + //Check rules + if len(handler.rules) != 0 { + violationOccured, err := handler.testRulesViolation(query) + if err != nil { + return ErrParseSqlRuleWhitelist + } + if violationOccured { + return ErrForbiddenSqlStructureWhitelist + } + } + return nil +} + +func (handler *WhitelistHandler) Reset() { + handler.queries = nil + handler.tables = nil + handler.rules = nil +} + +func (handler *WhitelistHandler) GetName() string { + return "Whitelist" +} + +func (handler *WhitelistHandler) AddQueries(queries []string) error { + for _, query := range queries { + handler.queries = append(handler.queries, query) + _, err := sqlparser.Parse(query) + if err != nil { + return ErrQuerySyntaxError + } + } + handler.queries = removeDuplicates(handler.queries) + return nil +} + +func (handler *WhitelistHandler) RemoveQueries(queries []string) { + for _, query := range handler.queries { + yes, index := contains(handler.queries, query) + if yes { + handler.queries = append(handler.queries[:index], handler.queries[index+1:]...) + } + } +} + +func (handler *WhitelistHandler) AddTables(tableNames []string) { + for _, tableName := range tableNames { + handler.tables = append(handler.tables, tableName) + } + handler.tables = removeDuplicates(handler.tables) +} + +func (handler *WhitelistHandler) RemoveTables(tableNames []string) { + for _, query := range tableNames { + yes, index := contains(handler.tables, query) + if yes { + handler.tables = append(handler.tables[:index], handler.tables[index+1:]...) + } + } +} + +func (handler *WhitelistHandler) AddRules(rules []string) error { + for _, rule := range rules { + handler.rules = append(handler.rules, rule) + _, err := sqlparser.Parse(rule) + if err != nil { + return ErrStructureSyntaxError + } + } + handler.rules = removeDuplicates(handler.rules) + return nil +} + +func (handler *WhitelistHandler) RemoveRules(rules []string) { + for _, rule := range rules { + yes, index := contains(handler.rules, rule) + if yes { + handler.rules = append(handler.rules[:index], handler.rules[index+1:]...) + } + } +} + +func (handler *WhitelistHandler) testRulesViolation(query string) (bool, error) { + if sqlparser.Preview(query) != sqlparser.StmtSelect { + return true, errors.New("non-select queries are not supported") + } + //parse one rule and get forbidden tables and columns for specific 'where' clause + var whereClause sqlparser.SQLNode + var tables sqlparser.TableExprs + var columns sqlparser.SelectExprs + //Parse each rule and then test query + for _, rule := range handler.rules { + parsedRule, err := sqlparser.Parse(rule) + if err != nil { + return true, err + } + switch parsedRule := parsedRule.(type) { + case *sqlparser.Select: + whereClause = parsedRule.Where.Expr + tables = parsedRule.From + columns = parsedRule.SelectExprs + dangerousSelect, err := handler.isDangerousSelect(query, whereClause, tables, columns) + if err != nil { + return true, err + } + if dangerousSelect { + return true, nil + } + case *sqlparser.Insert: + return true, ErrNotImplemented + default: + return true, ErrNotImplemented + } + _ = whereClause + _ = tables + _ = columns + } + return false, nil +} + +func (handler *WhitelistHandler) isDangerousSelect(selectQuery string, allowedWhere sqlparser.SQLNode, allowedTables sqlparser.TableExprs, allowedColumns sqlparser.SelectExprs) (bool, error) { + parsedSelectQuery, err := sqlparser.Parse(selectQuery) + if err != nil { + return true, err + } + evaluatedStmt := parsedSelectQuery.(*sqlparser.Select) + if strings.EqualFold(sqlparser.String(allowedWhere), sqlparser.String(evaluatedStmt.Where.Expr)) { + if handler.isAllowedTableAccess(evaluatedStmt.From, allowedTables) { + if handler.isAllowedColumnAccess(evaluatedStmt.SelectExprs, allowedColumns) { + return false, nil + } + } + } + return true, nil +} + +func (handler *WhitelistHandler) isAllowedTableAccess(tablesToEvaluate sqlparser.TableExprs, allowedTables sqlparser.TableExprs) bool { + accessOnlyToAllowedTables := true + for _, tableToEvaluate := range tablesToEvaluate { + for _, allowedTable := range allowedTables { + if !reflect.DeepEqual(tableToEvaluate.(*sqlparser.AliasedTableExpr).Expr, allowedTable.(*sqlparser.AliasedTableExpr).Expr) { + accessOnlyToAllowedTables = false + } + } + } + return accessOnlyToAllowedTables +} + +func (handler *WhitelistHandler) isAllowedColumnAccess(columnsToEvaluate sqlparser.SelectExprs, allowedColumns sqlparser.SelectExprs) bool { + if strings.EqualFold(sqlparser.String(allowedColumns), "*") { + return true + } + accessOnlyToAllowedColumns := true + for _, columnToEvaluate := range columnsToEvaluate { + for _, allowedColumn := range allowedColumns { + if !reflect.DeepEqual(columnToEvaluate, allowedColumn) { + accessOnlyToAllowedColumns = false + } + } + } + return accessOnlyToAllowedColumns +} diff --git a/cmd/acra_addzone/acra_addzone.go b/cmd/acra_addzone/acra_addzone.go index 2e4ad2151..5f8aa322b 100644 --- a/cmd/acra_addzone/acra_addzone.go +++ b/cmd/acra_addzone/acra_addzone.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/utils" "github.com/cossacklabs/acra/zone" "github.com/cossacklabs/themis/gothemis/keys" @@ -32,7 +33,7 @@ func main() { outputDir := flag.String("output_dir", keystore.DEFAULT_KEY_DIR_SHORT, "Folder where will be saved generated zone keys") fsKeystore := flag.Bool("fs", true, "Use filesystem key store") - cmd.SetLogLevel(cmd.LOG_VERBOSE) + logging.SetLogLevel(logging.LOG_VERBOSE) err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { @@ -49,11 +50,23 @@ func main() { } var keyStore keystore.KeyStore if *fsKeystore { - keyStore, err = keystore.NewFilesystemKeyStore(output) + masterKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + scellEncryptor, err := keystore.NewSCellKeyEncryptor(masterKey) + if err != nil { + log.WithError(err).Errorln("can't init scell encryptor") + os.Exit(1) + } + keyStore, err = keystore.NewFilesystemKeyStore(output, scellEncryptor) if err != nil { log.WithError(err).Errorln("can't create key store") os.Exit(1) } + } else { + panic("No more supported keystores") } id, publicKey, err := keyStore.GenerateZoneKey() if err != nil { diff --git a/cmd/acra_configui/acra_configui.go b/cmd/acra_configui/acra_configui.go index a5b008510..449bc5780 100644 --- a/cmd/acra_configui/acra_configui.go +++ b/cmd/acra_configui/acra_configui.go @@ -1,29 +1,91 @@ +// Copyright 2018, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( - "io/ioutil" + "errors" + "bytes" + "encoding/json" + "flag" + "fmt" + "github.com/cossacklabs/acra/cmd" + "github.com/cossacklabs/acra/logging" + "strings" + "crypto/subtle" + "encoding/base64" + "github.com/cossacklabs/acra/utils" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "html/template" + "io/ioutil" "net/http" + "os" "path/filepath" - "gopkg.in/yaml.v2" - "time" - "encoding/json" "strconv" - "bytes" - "os" - "fmt" - "flag" - log "github.com/sirupsen/logrus" - "github.com/cossacklabs/acra/utils" - "github.com/cossacklabs/acra/cmd" + "time" ) +var host *string +var port *int var acraHost *string var acraPort *int var debug *bool +var staticPath *string +var authMode *string +var parsedTemplate *template.Template +var err error +var configParamsBytes []byte + +var SERVICE_NAME = "acra_configui" +var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName(SERVICE_NAME) + +var ErrGetAuthDataFromAcraServer = errors.New("Wrong status for loadAuthData") + +const ( + HTTP_TIMEOUT = 5 +) +const ( + LINE_SEPARATOR = "\n" + + AUTH_FIELD_SEPARATOR = ":" + AUTH_FIELD_COUNT = 4 + AUTH_USER_NAME_IDX = 0 + AUTH_SALT_IDX = 1 + AUTH_ARGON2_PARAMS_IDX = 2 + AUTH_HASH_IDX = 3 + + ARGON2_PARAM_SEPARATOR = "," + ARGON2_PARAM_COUNT = 4 + ARGON2_TIME_IDX = 0 + ARGON2_MEMORY_IDX = 1 + ARGON2_THREADS_IDX = 2 + ARGON2_LENGTH_IDX = 3 + + ARGON2_TIME_INT = 32 + ARGON2_MEMORY_INT = 32 + ARGON2_THREADS_INT = 8 + ARGON2_LENGTH_INT = 32 + + UINT_BASE = 10 +) + +var authUsers = make(map[string]cmd.UserAuth) func check(e error) { if e != nil { + log.Error(e) panic(e) } } @@ -41,6 +103,8 @@ type configParamsYAML struct { Config []paramItem } +var outConfigParams configParamsYAML + type ConfigAcraServer struct { ProxyHost string `json:"host"` ProxyPort int `json:"port"` @@ -54,14 +118,18 @@ type ConfigAcraServer struct { } func SubmitSettings(w http.ResponseWriter, r *http.Request) { + log.Debugf("SubmitSettings request %v", r) if r.Method != "POST" { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorRequestMethodNotAllowed). + Errorln("Invalid request method") http.Error(w, "Invalid request method", http.StatusMethodNotAllowed) return } err := r.ParseForm() if err != nil { - log.WithError(err).Errorln("Request parsing failed") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseRequestData). + Errorln("Request parsing failed") http.Error(w, "Bad request", http.StatusBadRequest) return } @@ -81,13 +149,15 @@ func SubmitSettings(w http.ResponseWriter, r *http.Request) { } jsonToServer, err := json.Marshal(config) if err != nil { - log.WithError(err).Errorln("/setConfig json.Marshal failed") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantSetNewConfig). + Errorln("/setConfig json.Marshal failed") http.Error(w, err.Error(), http.StatusInternalServerError) return } req, err := http.NewRequest("POST", fmt.Sprintf("http://%v:%v/setConfig", *acraHost, *acraPort), bytes.NewBuffer(jsonToServer)) if err != nil { - log.WithError(err).Errorln("/setConfig http.NewRequest failed") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantSetNewConfig). + Errorln("/setConfig http.NewRequest failed") http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -95,7 +165,8 @@ func SubmitSettings(w http.ResponseWriter, r *http.Request) { client := &http.Client{} resp, err := client.Do(req) if err != nil { - log.WithError(err).Errorln("/setConfig client.Do failed") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantSetNewConfig). + Errorln("/setConfig client.Do failed") http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -105,48 +176,70 @@ func SubmitSettings(w http.ResponseWriter, r *http.Request) { w.Write(jsonToServer) } +func parseTemplate(staticPath string) (err error) { + log.Infof("Parsing template") + tplPath := filepath.Join(staticPath, "index.html") + tplPath, err = utils.AbsPath(tplPath) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadTemplate). + Errorf("No template file[%v]", tplPath) + return err + } + + parsedTemplate, err = template.ParseFiles(tplPath) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadTemplate). + Errorf("Error while parsing template - %v", tplPath) + return err + } + + return nil +} + func index(w http.ResponseWriter, r *http.Request) { + log.Debugf("Index request %v", r) w.Header().Set("Content-Security-Policy", "require-sri-for script style") - parsedTemplate, _ := template.ParseFiles(filepath.Join("static", "index.html")) - var outConfigParams configParamsYAML - configParamsYAML, err := ioutil.ReadFile("acraserver_config_vars.yaml") - check(err) // get current config var netClient = &http.Client{ - Timeout: time.Second * 5, + Timeout: time.Second * HTTP_TIMEOUT, } serverResponse, err := netClient.Get(fmt.Sprintf("http://%v:%v/getConfig", *acraHost, *acraPort)) if err != nil { - log.WithError(err).Errorln("AcraServer api error") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetCurrentConfig). + Errorln("AcraServer API error") http.Error(w, err.Error(), http.StatusInternalServerError) return } serverConfigDataJsonString, err := ioutil.ReadAll(serverResponse.Body) if err != nil { - log.Fatal(err.Error()) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetCurrentConfig). + Errorln("Can't read configuration") http.Error(w, err.Error(), http.StatusInternalServerError) return } var serverConfigData ConfigAcraServer err = json.Unmarshal(serverConfigDataJsonString, &serverConfigData) if err != nil { - log.WithError(err).Errorln("json.Unmarshal error") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetCurrentConfig). + Errorln("Can't unmarshal server config params") http.Error(w, err.Error(), http.StatusInternalServerError) return } // end get current config - err = yaml.Unmarshal(configParamsYAML, &outConfigParams) + err = yaml.Unmarshal(configParamsBytes, &outConfigParams) if err != nil { - log.Errorf("%v", utils.ErrorMessage("yaml.Unmarshal error", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetCurrentConfig). + Errorln("Can't unmarshal config params") http.Error(w, err.Error(), http.StatusInternalServerError) return } res, err := json.Marshal(outConfigParams) if err != nil { - log.Errorf("%v", utils.ErrorMessage("json.Marshal error", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetCurrentConfig). + Errorln("Can't marshal config params") http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -160,24 +253,171 @@ func index(w http.ResponseWriter, r *http.Request) { }) } +func BasicAuthHandler(handler http.HandlerFunc) http.HandlerFunc { + var realm = "AcraConfigUI" + + return func(w http.ResponseWriter, r *http.Request) { + if *authMode == "auth_on" || + (*authMode == "auth_off_local" && *host != "127.0.0.1" && *host != "localhost") { + + user, pass, basicOk := r.BasicAuth() + + if _, ok := authUsers[user]; !ok { + log.Warningf("BasicAuth: unknown user '%v'", user) + basicOk = false + } + + var newHash []byte + var authUserData cmd.UserAuth + var err error + if basicOk { + authUserData = authUsers[user] + newHash, err = cmd.HashArgon2(pass, authUserData.Salt, authUserData.Argon2Params) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantHashPassword). + Error("Error while hashing user password") + basicOk = false + } + } + if !basicOk || subtle.ConstantTimeCompare(newHash, authUserData.Hash) != 1 { + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm="%v"`, realm)) + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte(http.StatusText(http.StatusUnauthorized))) + return + } + } + handler(w, r) + } +} + +func ParseArgon2Params(authDataSting []byte) { + line := 0 + for _, authString := range strings.Split(string(authDataSting), LINE_SEPARATOR) { + authItem := strings.Split(authString, AUTH_FIELD_SEPARATOR) + line += 1 + if len(authItem) == AUTH_FIELD_COUNT { + decoded, err := base64.StdEncoding.DecodeString(string(authItem[AUTH_HASH_IDX])) + if err != nil { + log.WithError(err).Errorf("line[%v] DecodeString, user: %v", line, authItem[AUTH_USER_NAME_IDX]) + continue + } + argon2P := strings.Split(authItem[AUTH_ARGON2_PARAMS_IDX], ARGON2_PARAM_SEPARATOR) + if len(authItem) != ARGON2_PARAM_COUNT { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Errorf("line[%v] wrong number of argon2 params: got %v, expected %v", line, len(authItem), ARGON2_PARAM_COUNT) + continue + } + Time, err := strconv.ParseUint(argon2P[ARGON2_TIME_IDX], UINT_BASE, ARGON2_TIME_INT) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Errorf("line[%v] argon2 strconv.ParseUint(%v), user: %v", line, "Time", authItem[AUTH_USER_NAME_IDX]) + continue + } + Memory, err := strconv.ParseUint(argon2P[ARGON2_MEMORY_IDX], UINT_BASE, ARGON2_MEMORY_INT) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Errorf("line[%v] argon2 strconv.ParseUint(%v), user: %v", line, "Memory", authItem[AUTH_USER_NAME_IDX]) + continue + } + Threads, err := strconv.ParseUint(argon2P[ARGON2_THREADS_IDX], UINT_BASE, ARGON2_THREADS_INT) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Errorf("line[%v] argon2 strconv.ParseUint(%v), user: %v", line, "Threads", authItem[AUTH_USER_NAME_IDX]) + continue + } + Length, err := strconv.ParseUint(argon2P[ARGON2_LENGTH_IDX], UINT_BASE, ARGON2_LENGTH_INT) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Errorf("line[%v] argon2 strconv.ParseUint(%v), user: %v", line, "Length", authItem[AUTH_USER_NAME_IDX]) + continue + } + authUsers[authItem[AUTH_USER_NAME_IDX]] = cmd.UserAuth{Salt: authItem[AUTH_SALT_IDX], Hash: decoded, Argon2Params: cmd.Argon2Params{ + Time: uint32(Time), + Memory: uint32(Memory), + Threads: uint8(Threads), + Length: uint32(Length), + }} + } + } +} + +func loadAuthData() (err error) { + var netClient = &http.Client{ + Timeout: time.Second * HTTP_TIMEOUT, + } + serverResponse, err := netClient.Get(fmt.Sprintf("http://%v:%v/loadAuthData", *acraHost, *acraPort)) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetAuthData). + Error("Error while getting auth data from Acraserver") + return err + } + defer serverResponse.Body.Close() + if serverResponse.StatusCode != http.StatusOK { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetAuthData). + Errorf("Error while getting auth data from Acraserver, response status: %v", serverResponse.Status) + return ErrGetAuthDataFromAcraServer + } + authDataSting, err := ioutil.ReadAll(serverResponse.Body) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantParseAuthData). + Error("Error while reading auth data") + return err + } + ParseArgon2Params(authDataSting) + return +} + func main() { - port := flag.Int("port", 8000, "Port for configUI HTTP endpoint") - acraHost = flag.String("acraHost", "localhost", "Host for Acraserver HTTP endpoint or proxy") - acraPort = flag.Int("acraPort", 9292, "Port for Acraserver HTTP endpoint or proxy") + host = flag.String("host", cmd.DEFAULT_ACRA_CONFIGUI_HOST, "Host for configUI HTTP endpoint") + port = flag.Int("port", cmd.DEFAULT_ACRA_CONFIGUI_PORT, "Port for configUI HTTP endpoint") + loggingFormat := flag.String("logging_format", "plaintext", "Logging format: plaintext, json or CEF") + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + log.Infof("Starting service") + acraHost = flag.String("acra_host", "localhost", "Host for Acraserver HTTP endpoint or proxy") + acraPort = flag.Int("acra_port", cmd.DEFAULT_PROXY_API_PORT, "Port for Acraserver HTTP endpoint or proxy") + staticPath = flag.String("static_path", cmd.DEFAULT_ACRA_CONFIGUI_STATIC, "Path to static content") debug = flag.Bool("d", false, "Turn on debug logging") - flag.Parse() + authMode = flag.String("auth_mode", cmd.DEFAULT_ACRA_CONFIGUI_AUTH_MODE, "Mode for basic auth. Possible values: auth_on|auth_off_local|auth_off") + + err = cmd.Parse(DEFAULT_CONFIG_PATH) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadServiceConfig). + Errorln("Can't parse args") + os.Exit(1) + } + + // if log format was overridden + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + + log.Infof("Validating service configuration") if *debug { - cmd.SetLogLevel(cmd.LOG_DEBUG) + logging.SetLogLevel(logging.LOG_DEBUG) + } else { + logging.SetLogLevel(logging.LOG_VERBOSE) + } + + err = parseTemplate(*staticPath) + if err != nil { + os.Exit(1) + } + + if *authMode == "auth_off" { + log.Warningf("HTTP Basic Auth is turned off") } else { - cmd.SetLogLevel(cmd.LOG_VERBOSE) + log.Infof("HTTP Basic Auth mode: %v", *authMode) + err = loadAuthData() + if err != nil { + os.Exit(1) + } } - http.HandleFunc("/index.html", index) - http.HandleFunc("/", index) - http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static")))) - http.HandleFunc("/acraserver/submit_setting", SubmitSettings) - log.Info(fmt.Sprintf("AcraConfigUI is listening @ :%d with PID %d", *port, os.Getpid())) - err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil) + configParamsBytes = []byte(AcraServerCofig) + http.HandleFunc("/index.html", BasicAuthHandler(index)) + http.HandleFunc("/", BasicAuthHandler(index)) + http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(*staticPath)))) + http.HandleFunc("/acraserver/submit_setting", BasicAuthHandler(SubmitSettings)) + log.Infof("AcraConfigUI is listening @ %s:%d with PID %d", *host, *port, os.Getpid()) + err = http.ListenAndServe(fmt.Sprintf("%s:%d", *host, *port), nil) check(err) } diff --git a/cmd/acra_configui/acraserver_config_vars.yaml b/cmd/acra_configui/acraserver_config_vars.go similarity index 58% rename from cmd/acra_configui/acraserver_config_vars.yaml rename to cmd/acra_configui/acraserver_config_vars.go index 0215c3679..5e4f5d40d 100644 --- a/cmd/acra_configui/acraserver_config_vars.yaml +++ b/cmd/acra_configui/acraserver_config_vars.go @@ -1,14 +1,20 @@ -config: -# - -# name: host -# title: Host network address to listen for incoming connections from AcraProxy or via SSL -# value_type: string -# input_type: text -# - -# name: port -# title: Port for AcraServer to listen for incoming connections from AcraProxy or via SSL -# value_type: int8 -# input_type: number +// Copyright 2018, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +var AcraServerCofig = `config: - name: db_host title: Host for destination Postgres @@ -43,12 +49,6 @@ config: values: [true, false] labels: [Yes, No] input_type: radio - - - - name: server_id - title: ID to be sent in secure session - value_type: string - input_type: text - name: zonemode title: Turn on zone mode @@ -56,3 +56,4 @@ config: values: [true, false] labels: [Yes, No] input_type: radio +` diff --git a/cmd/acra_configui/static/index.html b/cmd/acra_configui/static/index.html index 1b3051bad..eea783272 100644 --- a/cmd/acra_configui/static/index.html +++ b/cmd/acra_configui/static/index.html @@ -76,7 +76,7 @@
diff --git a/cmd/acra_configui/static/js/main.js b/cmd/acra_configui/static/js/main.js index aea8731f5..6dc3fe6a6 100644 --- a/cmd/acra_configui/static/js/main.js +++ b/cmd/acra_configui/static/js/main.js @@ -1,52 +1,52 @@ -$(document).ready(function () { - $.views.settings.delimiters("{-", "-}"); - var options = []; - $.each(configParams.Config, function (i, item) { - options.push(item); - }); - var tpl = $($.templates('#settingsTplRow').render({ - options: options - })); - tpl.appendTo($('#v-pills-settings')); - - // set checkox values - $.each(configParams['Config'], function (i, item) { - if (item.input_type == 'radio') { - if (currentConfig[item.name] == undefined) { - $('#v-pills-settings').find('input[type="radio"][name="' + item.name + '"][value="' + item.value + '"]').attr('checked', 'checked'); - } - else { - var v = currentConfig[item.name] ? 1 : 0; - $('#v-pills-settings').find('input[type="radio"][name="' + item.name + '"][value="' + currentConfig[item.name] + '"]').attr('checked', 'checked'); - } - } - else { - $('#v-pills-settings').find('input[name="' + item.name + '"]').val(currentConfig[item.name]); - } - }); - - $('#v-pills-tab a').on('click', function (e) { - e.preventDefault(); - $(this).tab('show'); - }) -}); - -var save = function () { - var data = {}; - $.each(configParams['Config'], function (i, item) { - if (item.input_type == 'radio') { - data[item.name] = $('#v-pills-settings').find('input:checked[type="radio"][name="' + item.name + '"]').val(); - } - else { - data[item.name] = $('#v-pills-settings').find('input[name="' + item.name + '"]').val(); - } - }); - - $.ajax({ - method: 'POST', - url: "/acraserver/submit_setting", - data: data - }).done(function () { - $(this).addClass("done"); - }); -}; +$(document).ready(function () { + $.views.settings.delimiters("{-", "-}"); + var options = []; + $.each(configParams.Config, function (i, item) { + options.push(item); + }); + var tpl = $($.templates('#settingsTplRow').render({ + options: options + })); + tpl.appendTo($('#v-pills-settings')); + + // set checkox values + $.each(configParams['Config'], function (i, item) { + if (item.input_type == 'radio') { + if (currentConfig[item.name] == undefined) { + $('#v-pills-settings').find('input[type="radio"][name="' + item.name + '"][value="' + item.value + '"]').attr('checked', 'checked'); + } + else { + var v = currentConfig[item.name] ? 1 : 0; + $('#v-pills-settings').find('input[type="radio"][name="' + item.name + '"][value="' + currentConfig[item.name] + '"]').attr('checked', 'checked'); + } + } + else { + $('#v-pills-settings').find('input[name="' + item.name + '"]').val(currentConfig[item.name]); + } + }); + + $('#v-pills-tab a').on('click', function (e) { + e.preventDefault(); + $(this).tab('show'); + }) +}); + +var save = function () { + var data = {}; + $.each(configParams['Config'], function (i, item) { + if (item.input_type == 'radio') { + data[item.name] = $('#v-pills-settings').find('input:checked[type="radio"][name="' + item.name + '"]').val(); + } + else { + data[item.name] = $('#v-pills-settings').find('input[name="' + item.name + '"]').val(); + } + }); + + $.ajax({ + method: 'POST', + url: "/acraserver/submit_setting", + data: data + }).done(function () { + $(this).addClass("done"); + }); +}; diff --git a/cmd/acra_genauth/acra_genauth.go b/cmd/acra_genauth/acra_genauth.go new file mode 100644 index 000000000..034d6b234 --- /dev/null +++ b/cmd/acra_genauth/acra_genauth.go @@ -0,0 +1,223 @@ +// Copyright 2018, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "flag" + "fmt" + "github.com/cossacklabs/acra/cmd" + "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" + "github.com/cossacklabs/themis/gothemis/cell" + log "github.com/sirupsen/logrus" + "io/ioutil" + "os" + "strings" +) + +type HashedPasswords map[string]string + +const ( + AuthFieldSeparator = ":" + AuthArgon2ParamSeparator = "," + LineSeparator = "\n" + SaltLength = 16 + AuthFieldCount = 4 + Space = " " +) + +func (hp HashedPasswords) Bytes() (passwordBytes []byte) { + passwordBytes = []byte{} + for name, hash := range hp { + passwordBytes = append(passwordBytes, []byte(name+AuthFieldSeparator+hash+LineSeparator)...) + } + return passwordBytes +} + +func (hp HashedPasswords) WriteToFile(file string, keystore *keystore.FilesystemKeyStore) error { + key, err := keystore.GetAuthKey(false) + if err != nil { + return err + } + SecureCell := cell.New(key, cell.CELL_MODE_SEAL) + crypted, _, err := SecureCell.Protect(hp.Bytes(), nil) + if err != nil { + return err + } + return ioutil.WriteFile(file, crypted, 0600) +} + +func (hp HashedPasswords) SetPassword(name, password string) (err error) { + if len(password) == 0 { + return errors.New("passwords is empty") + } + salt := cmd.RandomStringBytes(SaltLength) + argon2Params := cmd.InitArgon2Params() + hashBytes, err := cmd.HashArgon2(password, salt, argon2Params) + if err != nil { + return err + } + if err != nil { + return err + } + a := cmd.UserAuth{Salt: salt, Hash: hashBytes, Argon2Params: argon2Params} + hp[name] = a.UserAuthString(AuthFieldSeparator, AuthArgon2ParamSeparator) + return nil +} + +func ParseHtpasswdFile(file string, keystore *keystore.FilesystemKeyStore) (passwords HashedPasswords, err error) { + htpasswdBytes, err := ioutil.ReadFile(file) + if err != nil { + return + } + key, err := keystore.GetAuthKey(false) + if err != nil { + return + } + SecureCell := cell.New(key, cell.CELL_MODE_SEAL) + authData, err := SecureCell.Unprotect(htpasswdBytes, nil, nil) + if err != nil { + return + } + return ParseHtpasswd(authData) +} + +func ParseHtpasswd(htpasswdBytes []byte) (passwords HashedPasswords, err error) { + lines := strings.Split(string(htpasswdBytes), LineSeparator) + passwords = make(map[string]string) + for index, line := range lines { + line = strings.Trim(line, Space) + if len(line) == 0 { + continue + } + parts := strings.Split(line, AuthFieldSeparator) + if len(parts) != AuthFieldCount { + err = errors.New(fmt.Sprintf("wrong line no. %d, unexpected number (%v) of splitted parts split by %v", index+1, len(parts), AuthFieldSeparator)) + return + } + for i, part := range parts { + parts[i] = strings.Trim(part, Space) + } + _, alreadyExists := passwords[parts[0]] + if alreadyExists { + err = errors.New(fmt.Sprintf("wrong line no. %d, user (%v) already defined", index, parts[0])) + return + } + passwords[parts[0]] = strings.Join(parts[1:AuthFieldCount], AuthFieldSeparator) + } + return +} + +func RemoveUser(file, user string, keystore *keystore.FilesystemKeyStore) error { + passwords, err := ParseHtpasswdFile(file, keystore) + if err != nil { + return err + } + _, ok := passwords[user] + if !ok { + return errors.New("user not found in file") + } + delete(passwords, user) + return passwords.WriteToFile(file, keystore) +} + +func SetPassword(file, name, password string, keystore *keystore.FilesystemKeyStore) error { + _, err := os.Stat(file) + passwords := HashedPasswords(map[string]string{}) + if err == nil { + passwords, err = ParseHtpasswdFile(file, keystore) + if err != nil { + return err + } + } + err = passwords.SetPassword(name, password) + if err != nil { + return err + } + return passwords.WriteToFile(file, keystore) +} + +func main() { + set := flag.Bool("set", false, "Add/update password for user") + remove := flag.Bool("remove", false, "Remove user") + user := flag.String("user", "", "User") + password := flag.String("password", "", "Password") + filePath := flag.String("file", cmd.DEFAULT_ACRA_AUTH_PATH, "Auth file") + keysDir := flag.String("keys_dir", keystore.DEFAULT_KEY_DIR_SHORT, "Folder from which will be loaded keys") + debug := flag.Bool("d", false, "Turn on debug logging") + flag.Parse() + flags := []*bool{set, remove} + + if *debug { + logging.SetLogLevel(logging.LOG_DEBUG) + } else { + logging.SetLogLevel(logging.LOG_VERBOSE) + } + + masterKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + encryptor, err := keystore.NewSCellKeyEncryptor(masterKey) + if err != nil { + log.WithError(err).Errorln("can't initialize scell encryptor") + os.Exit(1) + } + keyStore, err := keystore.NewFilesystemKeyStore(*keysDir, encryptor) + if err != nil { + log.WithError(err).Errorln("NewFilesystemKeyStore") + os.Exit(1) + } + + n := 0 + for _, o := range flags { + if *o { + n += 1 + if n > 1 { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongParam).Errorln("Too many options, use one of --set or --remove") + os.Exit(1) + } + } + } + + if *user == "" { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongParam).Errorln("Empty user name/login") + flag.Usage() + os.Exit(1) + } + + if *set { + if *password == "" { + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongParam).Errorln("Empty password") + flag.Usage() + os.Exit(1) + } + err := SetPassword(*filePath, *user, *password, keyStore) + if err != nil { + log.WithError(err).Errorln("SetPassword failed") + os.Exit(1) + } + } + if *remove { + err := RemoveUser(*filePath, *user, keyStore) + if err != nil { + log.WithError(err).Errorln("RemoveUser failed") + os.Exit(1) + } + } + +} diff --git a/cmd/acra_genkeys/acra_genkeys.go b/cmd/acra_genkeys/acra_genkeys.go index 1f01d3cc1..4b8b5fa58 100644 --- a/cmd/acra_genkeys/acra_genkeys.go +++ b/cmd/acra_genkeys/acra_genkeys.go @@ -17,8 +17,10 @@ import ( "flag" "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/utils" log "github.com/sirupsen/logrus" + "io/ioutil" "os" ) @@ -27,12 +29,15 @@ var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName("acra_genkeys") func main() { clientId := flag.String("client_id", "client", "Client id") - acraproxy := flag.Bool("acraproxy", false, "Create keypair only for acraproxy") - acraserver := flag.Bool("acraserver", false, "Create keypair only for acraserver") + acraproxy := flag.Bool("acraproxy", false, "Create keypair for acraproxy only") + acraserver := flag.Bool("acraserver", false, "Create keypair for acraserver only") dataKeys := flag.Bool("storage", false, "Create keypair for data encryption/decryption") + basicauth := flag.Bool("basicauth", false, "Create symmetric key for acra_configui's basic auth db") outputDir := flag.String("output", keystore.DEFAULT_KEY_DIR_SHORT, "Folder where will be saved keys") + outputPublicKey := flag.String("output_public", keystore.DEFAULT_KEY_DIR_SHORT, "Folder where will be saved public key") + masterKey := flag.String("master_key", "", "Generate new random master key and save to file") - cmd.SetLogLevel(cmd.LOG_VERBOSE) + logging.SetLogLevel(logging.LOG_VERBOSE) err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { @@ -42,7 +47,37 @@ func main() { cmd.ValidateClientId(*clientId) - store, err := keystore.NewFilesystemKeyStore(*outputDir) + if *masterKey != "" { + newKey, err := keystore.GenerateSymmetricKey() + if err != nil { + panic(err) + } + if err := ioutil.WriteFile(*masterKey, newKey, 0600); err != nil { + panic(err) + } + os.Exit(0) + } + + symmetricKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + if err == keystore.ErrEmptyMasterKey { + log.Infof("You must pass master key via %v environment variable", keystore.ACRA_MASTER_KEY_VAR_NAME) + os.Exit(1) + } + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + scellEncryptor, err := keystore.NewSCellKeyEncryptor(symmetricKey) + if err != nil { + log.WithError(err).Errorln("can't init scell encryptor") + os.Exit(1) + } + var store keystore.KeyStore + if *outputPublicKey != *outputDir { + store, err = keystore.NewFilesystemKeyStoreTwoPath(*outputDir, *outputPublicKey, scellEncryptor) + } else { + store, err = keystore.NewFilesystemKeyStore(*outputDir, scellEncryptor) + } if err != nil { panic(err) } @@ -62,6 +97,11 @@ func main() { if err != nil { panic(err) } + } else if *basicauth { + _, err = store.GetAuthKey(true) + if err != nil { + panic(err) + } } else { err = store.GenerateProxyKeys([]byte(*clientId)) if err != nil { diff --git a/cmd/acra_genpoisonrecord/Description.md b/cmd/acra_genpoisonrecord/Description.md deleted file mode 100644 index 8c92f2831..000000000 --- a/cmd/acra_genpoisonrecord/Description.md +++ /dev/null @@ -1,55 +0,0 @@ -# Usage -## With zones -The same like for correct acra structs: -* add zone like -``` -[acra]$ go build acra_addzone -[acra]$ ./acra_addzone -{"id":"ZXChVYmCtQgasDlqtrz","public_key":"VUVDMgAAAC3e8NhhAjK+wyOntdyqlPg3c89Vu5z3JskxuxgAQVynNOLnel4c"} -``` -* save decoded from base64 public key in file (for example ZXChVYmCtQgasDlqtrz.pub). For example: -``` -echo VUVDMgAAAC3e8NhhAjK+wyOntdyqlPg3c89Vu5z3JskxuxgAQVynNOLnel4c | base64 --decode > ZXChVYmCtQgasDlqtrz.pub -``` -* create poison record using zone public key -``` -go build acra_genpoisonrecord -[acra]$ ./acra_genpoisonrecord -acra_public=/path/to/ZXChVYmCtQgasDlqtrz.pub -hSD7VUVDMgAAAC0GPZUAAtxiVBj7+LzTz5+qDQLuIF4MNIPpvuNOoGx5pMFHSmSRICcEJlQAAAAAAQFADAAAABAAAAAgAAAA+HNqMryNiASoXhFJdmkDieAWlfjBN7sOBbj4s96uS0i2PnKT9I9powzVn4CfHzFuCSNJceusSDV14GO1dwAAAAAAAAAAAQFADAAAABAAAABLAAAAofsRDZjjbVuC6pELSJEKLYoKEpYFk6xPQDKQLVFocUgTMM9gVPtKKVkr4AFH2lBTbG3+7+3b9Ebrczl8VLScsF2HDfOFQ2Oo0eUVJ+5d82SSTFmTjJEVOJ+XgXXRyI5bQamIwoYEhA== -``` -* Insert into db decoded from base64 this poison record like acrastruct - -## Without zones -* create key pair for acra: -``` -go build acra_genkeys -./acra_genkeys -client_id=test_server -acraserver -``` -* create poison record using this public key -``` -[acra]$ ./acra_genpoisonrecord -acra_public=.acrakeys/test_server.pub -hSD7VUVDMgAAAC373NQJAz5XMsVP3jXLFkFwfBb7H4NjxL6REJbeNZx/7blJodPfICcEJlQAAAAAAQFADAAAABAAAAAgAAAA6YxpknqByuENYMI9rv2U2AMJNTvmqEv+cro8yWTiQ7vGv/B4fy3Ehv0gruPNEdXGsEYNd654+So+ybg6WQAAAAAAAAAAAQFADAAAABAAAAAtAAAA52Ytsk+bGwXy6UxwMvLIyAFhq/3vzOdxZekHkTeRsTK17GAbnOKQBe3U0IHBvbStzVBjYeidNjW4vQxHYXSUzqHlG9kZm/Wp7A== -``` - -## Optional args -### Poison key -First run of acra or acra_genpoisonrecord will create `poison_key` - binary file with 32 -byte key that will used like identifier of poison record. As default it's `.acrakeys/poison_key` -or you can explicitly pass another value for acra like `-poison_key=/path/to/key` -and the same for acra_genpoisonrecord `-poison_key=/path/to/key`. Key will automatically -generated key if he isn't exists. - -### Data length -Optionally you can choose with which data length generate poison record (default -raw data length 1..100 bytes) with option `-data_length=n`. -This option for case when you have data with specific length and for similarity -you can explicitly pass this length -``` -Usage of ./acra_genpoisonrecord: - -acra_public string - path to acra public key to use - -data_length int - length of random data for data block in acrastruct. -1 is random in range 1..100 (default -1) - -poison_key string - path to file with poison key (default ".acrakeys/poison_key") -``` \ No newline at end of file diff --git a/cmd/acra_genpoisonrecord/acra_genpoisonrecord.go b/cmd/acra_genpoisonrecord/acra_genpoisonrecord.go index e97232ed2..893e7feb1 100644 --- a/cmd/acra_genpoisonrecord/acra_genpoisonrecord.go +++ b/cmd/acra_genpoisonrecord/acra_genpoisonrecord.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/poison" "github.com/cossacklabs/acra/utils" log "github.com/sirupsen/logrus" @@ -32,7 +33,7 @@ func main() { keysDir := flag.String("keys_dir", keystore.DEFAULT_KEY_DIR_SHORT, "Folder from which will be loaded keys") dataLength := flag.Int("data_length", poison.DEFAULT_DATA_LENGTH, fmt.Sprintf("Length of random data for data block in acrastruct. -1 is random in range 1..%v", poison.MAX_DATA_LENGTH)) - cmd.SetLogLevel(cmd.LOG_DISCARD) + logging.SetLogLevel(logging.LOG_DISCARD) err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { @@ -40,7 +41,17 @@ func main() { os.Exit(1) } - store, err := keystore.NewFilesystemKeyStore(*keysDir) + masterKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + scellEncryptor, err := keystore.NewSCellKeyEncryptor(masterKey) + if err != nil { + log.WithError(err).Errorln("can't init scell encryptor") + os.Exit(1) + } + store, err := keystore.NewFilesystemKeyStore(*keysDir, scellEncryptor) if err != nil { log.WithError(err).Errorln("can't initialize key store") os.Exit(1) diff --git a/cmd/acra_rollback/Readme.md b/cmd/acra_rollback/Readme.md deleted file mode 100644 index cfdfaed6f..000000000 --- a/cmd/acra_rollback/Readme.md +++ /dev/null @@ -1,23 +0,0 @@ -#Decrypting acrastructs encrypted without zones -You need pass as args: -* select sql query for fetching data from db `-select "select data from data_table;"` -* insert sql query with placeholder `$1` in which place will be inserted data or -binded for inserting directly to db (if you will use `-execute` param) `-insert 'insert into test_insert(data) values($1);` -* client id for finding key `-client_id=onekey` -* connection string that will be used to connect to db `-connection_string="dbname=some_database user=postgres password=postgres host=127.0.0.1 port=5432"` -Script will search key in `.acrakeys` folder with name , generate sql -insert queries to file `decrypted.sql`. If you use `-execute` arg, script will -insert data to db too. If you need just insert to db without generating output file, pass empty filename like `-output=""` -``` -./acra_rollback -select "select data from data_table;" -insert 'insert into test_insert(data) values($1);' -connection_string="dbname=some_database user=postgres password=postgres host=127.0.0.1 port=5432" -client_id=onekey -``` - -#Decrypting acrastructs encrypted with zones -The same but: -* use `-zonemode` param -* you don't need pass client id -* your SELECT sql query must fetch zone and data from db and zone should be first `-select "select zone, data from data_table;"` -* all zone private keys should be placed in keys dir (.acrakeys default) -``` -./acra_rollback -select "select zone, data from data_table;" -insert 'insert into test_insert(data) values($1);' -connection_string="dbname=some_database user=postgres password=postgres host=127.0.0.1 port=5432" -zonemode -``` diff --git a/cmd/acra_rollback/acra_rollback.go b/cmd/acra_rollback/acra_rollback.go index d180ae073..bcd35ea95 100644 --- a/cmd/acra_rollback/acra_rollback.go +++ b/cmd/acra_rollback/acra_rollback.go @@ -1,3 +1,5 @@ +// +build go1.8 + // Copyright 2016, Cossack Labs Limited // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,16 +22,20 @@ import ( "encoding/hex" "flag" "fmt" + "os" + "strings" + "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/decryptor/base" "github.com/cossacklabs/acra/decryptor/postgresql" "github.com/cossacklabs/acra/keystore" "github.com/cossacklabs/acra/utils" "github.com/cossacklabs/themis/gothemis/keys" + //_ "github.com/ziutek/mymysql/godrv" + "github.com/cossacklabs/acra/logging" + _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" log "github.com/sirupsen/logrus" - "os" - "strings" ) // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default @@ -44,6 +50,12 @@ type BinaryEncoder interface { Encode([]byte) string } +type MysqlEncoder struct{} + +func (e *MysqlEncoder) Encode(data []byte) string { + return fmt.Sprintf("X'%s'", hex.EncodeToString(data)) +} + type EscapeEncoder struct{} func (e *EscapeEncoder) Encode(data []byte) string { @@ -144,13 +156,15 @@ func main() { clientId := flag.String("client_id", "", "Client id should be name of file with private key") connectionString := flag.String("connection_string", "", "Connection string for db") sqlSelect := flag.String("select", "", "Query to fetch data for decryption") - sqlInsert := flag.String("insert", "", "Query for insert decrypted data with placeholders (pg: $n)") + sqlInsert := flag.String("insert", "", "Query for insert decrypted data with placeholders (pg: $n, mysql: ?)") withZone := flag.Bool("zonemode", false, "Turn on zone mode") outputFile := flag.String("output_file", "decrypted.sql", "File for store inserts queries") execute := flag.Bool("execute", false, "Execute inserts") escapeFormat := flag.Bool("escape", false, "Escape bytea format") + useMysql := flag.Bool("mysql", false, "Handle MySQL connections") + usePostgresql := flag.Bool("postgresql", false, "Handle Postgresql connections") - cmd.SetLogLevel(cmd.LOG_VERBOSE) + logging.SetLogLevel(logging.LOG_VERBOSE) err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { @@ -158,6 +172,24 @@ func main() { os.Exit(1) } + twoDrivers := *useMysql && *usePostgresql + noDrivers := !(*useMysql || *usePostgresql) + if twoDrivers || noDrivers { + log.Errorln("you must pass only --mysql or --postgresql (one required)") + os.Exit(1) + } + if *useMysql { + PLACEHOLDER = "?" + } + + dbDriverName := "postgres" + if *useMysql { + // https://github.com/ziutek/mymysql + //dbDriverName = "mymysql" + // https://github.com/go-sql-driver/mysql/ + dbDriverName = "mysql" + } + cmd.ValidateClientId(*clientId) if *connectionString == "" { @@ -182,12 +214,22 @@ func main() { log.Errorln("output_file missing or execute flag") os.Exit(1) } - keystorage, err := keystore.NewFilesystemKeyStore(absKeysDir) + masterKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + scellEncryptor, err := keystore.NewSCellKeyEncryptor(masterKey) + if err != nil { + log.WithError(err).Errorln("can't init scell encryptor") + os.Exit(1) + } + keystorage, err := keystore.NewFilesystemKeyStore(absKeysDir, scellEncryptor) if err != nil { log.WithError(err).Errorln("can't create key store") os.Exit(1) } - db, err := sql.Open("postgres", *connectionString) + db, err := sql.Open(dbDriverName, *connectionString) if err != nil { log.WithError(err).Errorln("can't connect to db") os.Exit(1) @@ -207,10 +249,14 @@ func main() { executors := list.New() if *outputFile != "" { - if *escapeFormat { - executors.PushFront(NewWriteToFileExecutor(*outputFile, *sqlInsert, &EscapeEncoder{})) + if *useMysql { + executors.PushFront(NewWriteToFileExecutor(*outputFile, *sqlInsert, &MysqlEncoder{})) } else { - executors.PushFront(NewWriteToFileExecutor(*outputFile, *sqlInsert, &HexEncoder{})) + if *escapeFormat { + executors.PushFront(NewWriteToFileExecutor(*outputFile, *sqlInsert, &EscapeEncoder{})) + } else { + executors.PushFront(NewWriteToFileExecutor(*outputFile, *sqlInsert, &HexEncoder{})) + } } } if *execute { diff --git a/cmd/acraproxy/acraproxy.go b/cmd/acraproxy/acraproxy.go index 3314795b7..b19d0bab1 100644 --- a/cmd/acraproxy/acraproxy.go +++ b/cmd/acraproxy/acraproxy.go @@ -31,12 +31,14 @@ import ( "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" "github.com/cossacklabs/acra/utils" ) // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default -var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName("acraproxy") +var SERVICE_NAME = "acraproxy" +var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName(SERVICE_NAME) func handleClientConnection(config *Config, connection net.Conn) { defer connection.Close() @@ -44,20 +46,23 @@ func handleClientConnection(config *Config, connection net.Conn) { if !(config.disableUserCheck) { host, port, err := net.SplitHostPort(connection.RemoteAddr().String()) if nil != err { - log.WithError(err).Errorln("can't parse client remote address") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Can't parse client remote address") return } if host == "127.0.0.1" { netstat, err := exec.Command("sh", "-c", "netstat -atlnpe | awk '/:"+port+" */ {print $7}'").Output() if nil != err { - log.WithError(err).Errorln("can't get owner UID of localhost client connection") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Can't get owner UID of localhost client connection") return } parsedNetstat := strings.Split(string(netstat), "\n") correctPeer := false userId, err := user.Current() if nil != err { - log.WithError(err).Errorln("can't get current user UID") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Can't get current user UID") return } log.Infof("%v\ncur_user=%v", parsedNetstat, userId.Uid) @@ -68,7 +73,8 @@ func handleClientConnection(config *Config, connection net.Conn) { } } if !correctPeer { - log.Errorln("client application and ssproxy need to be start from different users") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Client application and ssproxy need to be start from different users") return } } @@ -76,7 +82,8 @@ func handleClientConnection(config *Config, connection net.Conn) { acraConn, err := network.Dial(config.AcraConnectionString) if err != nil { - log.WithError(err).Errorln("can't connect to acra") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Can't connect to acra") return } defer acraConn.Close() @@ -84,7 +91,8 @@ func handleClientConnection(config *Config, connection net.Conn) { acraConn.SetDeadline(time.Now().Add(time.Second * 2)) acraConnWrapped, err := config.ConnectionWrapper.WrapClient(config.ClientId, acraConn) if err != nil { - log.WithError(err).Errorln("can't wrap connection") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection). + Errorln("Can't wrap connection") return } acraConn.SetDeadline(time.Time{}) @@ -96,15 +104,18 @@ func handleClientConnection(config *Config, connection net.Conn) { go network.Proxy(acraConnWrapped, connection, fromAcraErrCh) select { case err = <-toAcraErrCh: - log.Debugln("error from connection with client") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Error from connection with client") case err = <-fromAcraErrCh: - log.Debugln("error from connection with acra") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Error from connection with acra") } if err != nil { if err == io.EOF { - log.Debugln("connection closed") + log.Debugln("Connection closed") } else { - log.WithError(err).Errorln("proxy error") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Proxy error") } return } @@ -122,16 +133,20 @@ type Config struct { } func main() { + loggingFormat := flag.String("logging_format", "plaintext", "Logging format: plaintext, json or CEF") + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + log.Infof("Starting service") + keysDir := flag.String("keys_dir", keystore.DEFAULT_KEY_DIR_SHORT, "Folder from which will be loaded keys") clientId := flag.String("client_id", "", "Client id") acraHost := flag.String("acra_host", "", "IP or domain to acra daemon") acraCommandsPort := flag.Int("acra_commands_port", cmd.DEFAULT_ACRA_API_PORT, "Port of acra http api") acraPort := flag.Int("acra_port", cmd.DEFAULT_ACRA_PORT, "Port of acra daemon") acraId := flag.String("acra_id", "acra_server", "Expected id from acraserver for Secure Session") - verbose := flag.Bool("v", false, "Log to stdout") + verbose := flag.Bool("v", false, "Log to stderr") port := flag.Int("port", cmd.DEFAULT_PROXY_PORT, "Port fo acraproxy") commandsPort := flag.Int("command_port", cmd.DEFAULT_PROXY_API_PORT, "Port for acraproxy http api") - withZone := flag.Bool("zonemode", false, "Turn on zone mode") + enableHTTPApi := flag.Bool("enable_http_api", false, "Enable HTTP API") disableUserCheck := flag.Bool("disable_user_check", false, "Disable checking that connections from app running from another user") useTls := flag.Bool("tls", false, "Use tls to encrypt transport between acraserver and acraproxy/client") tlsCA := flag.String("tls_ca", "", "Path to root certificate") @@ -146,10 +161,15 @@ func main() { err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { - log.WithError(err).Errorln("can't parse args") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadServiceConfig). + Errorln("Can't parse args") os.Exit(1) } + // if log format was overridden + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + log.Infof("Validating service configuration") + if *port != cmd.DEFAULT_PROXY_PORT { *connectionString = network.BuildConnectionString(cmd.DEFAULT_PROXY_CONNECTION_PROTOCOL, cmd.DEFAULT_PROXY_HOST, *port, "") } @@ -158,15 +178,17 @@ func main() { } if *acraHost == "" && *acraConnectionString == "" { - log.Errorln("you must pass acra_host or acra_connection_string parameter") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorln("Configuration error: you must pass acra_host or acra_connection_string parameter") os.Exit(1) } if *acraHost != "" { *acraConnectionString = network.BuildConnectionString(cmd.DEFAULT_ACRA_CONNECTION_PROTOCOL, *acraHost, *acraPort, "") } - if *withZone { + if *enableHTTPApi { if *acraHost == "" && *acraApiConnectionString == "" { - log.Errorln("you must pass acra_host or acra_api_connection_string parameter") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorln("Configuration error: you must pass acra_host or acra_api_connection_string parameter") os.Exit(1) } if *acraHost != "" { @@ -176,120 +198,150 @@ func main() { cmd.ValidateClientId(*clientId) + log.Infof("Reading keys") clientPrivateKey := fmt.Sprintf("%v%v%v", *keysDir, string(os.PathSeparator), *clientId) serverPublicKey := fmt.Sprintf("%v%v%v_server.pub", *keysDir, string(os.PathSeparator), *clientId) exists, err := utils.FileExists(clientPrivateKey) if !exists { - log.Errorf("acraproxy private key %s doesn't exists", clientPrivateKey) + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorf("Configuration error: acraproxy private key %s doesn't exists", clientPrivateKey) os.Exit(1) } if err != nil { - log.Errorf("can't check is exists acraproxy private key %v, got error - %v", clientPrivateKey, err) + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorf("Configuration error: can't check is exists acraproxy private key %v, got error - %v", clientPrivateKey, err) os.Exit(1) } exists, err = utils.FileExists(serverPublicKey) if !exists { - log.Errorf("acraserver public key %s doesn't exists", serverPublicKey) + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorf("Configuration error: acraserver public key %s doesn't exists", serverPublicKey) os.Exit(1) } if err != nil { - log.Errorf("can't check is exists acraserver public key %v, got error - %v", serverPublicKey, err) + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorf("Configuration error: can't check is exists acraserver public key %v, got error - %v", serverPublicKey, err) os.Exit(1) } if *verbose { - cmd.SetLogLevel(cmd.LOG_VERBOSE) + logging.SetLogLevel(logging.LOG_VERBOSE) } else { - cmd.SetLogLevel(cmd.LOG_DISCARD) + logging.SetLogLevel(logging.LOG_DISCARD) } if runtime.GOOS != "linux" { *disableUserCheck = true } - keyStore, err := keystore.NewProxyFileSystemKeyStore(*keysDir, []byte(*clientId)) + log.Infof("Initializing keystore") + masterKey, err := keystore.GetMasterKeyFromEnvironment() + if err != nil { + log.WithError(err).Errorln("can't load master key") + os.Exit(1) + } + scellEncryptor, err := keystore.NewSCellKeyEncryptor(masterKey) if err != nil { - log.WithError(err).Errorln("can't initialize keystore") + log.WithError(err).Errorln("can't init scell encryptor") os.Exit(1) } + keyStore, err := keystore.NewProxyFileSystemKeyStore(*keysDir, []byte(*clientId), scellEncryptor) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantInitKeyStore). + Errorln("Can't initialize keystore") + os.Exit(1) + } + + log.Debugf("Start listening connections") config := &Config{KeyStore: keyStore, KeysDir: *keysDir, ClientId: []byte(*clientId), AcraConnectionString: *acraConnectionString, ConnectionString: *connectionString, AcraId: []byte(*acraId), disableUserCheck: *disableUserCheck} listener, err := network.Listen(*connectionString) if err != nil { - log.WithError(err).Errorln("can't start listen connections") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartListenConnections). + Errorln("Can't start listen connections") os.Exit(1) } defer listener.Close() + log.Debugf("Registering process signal handlers") sigHandler, err := cmd.NewSignalHandler([]os.Signal{os.Interrupt, syscall.SIGTERM}) if err != nil { - log.WithError(err).Errorln("can't register SIGINT handler") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantRegisterSignalHandler). + Errorln("Can't register SIGINT handler") os.Exit(1) } go sigHandler.Register() sigHandler.AddListener(listener) + if *useTls { - log.Infoln("use TLS transport wrapper") + log.Infof("Selecting transport: use TLS transport wrapper") tlsConfig, err := network.NewTLSConfig(*tlsSNI, *tlsCA, *tlsKey, *tlsCert) if err != nil { - log.WithError(err).Errorln("can't get config for TLS") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't get config for TLS") os.Exit(1) } config.ConnectionWrapper, err = network.NewTLSConnectionWrapper(nil, tlsConfig) if err != nil { - log.WithError(err).Errorln("can't initialize tls connection wrapper") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't initialize TLS connection wrapper") os.Exit(1) } } else if *noEncryption { - log.Infoln("use raw transport wrapper") + log.Infof("Selecting transport: use raw transport wrapper") config.ConnectionWrapper = &network.RawConnectionWrapper{ClientId: []byte(*clientId)} } else { - log.Infoln("use Secure Session transport wrapper") + log.Infof("Selecting transport: use Secure Session transport wrapper") config.ConnectionWrapper, err = network.NewSecureSessionConnectionWrapper(keyStore) if err != nil { - log.WithError(err).Errorln("can't initialize secure session connection wrapper") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't initialize secure session connection wrapper") os.Exit(1) } } - if *withZone { + if *enableHTTPApi { go func() { // copy config and replace ports commandsConfig := *config commandsConfig.AcraConnectionString = *acraApiConnectionString - log.Infof("start listening http api %s", *connectionAPIString) + log.Infof("Start listening http API: %s", *connectionAPIString) commandsListener, err := network.Listen(*connectionAPIString) if err != nil { - log.WithError(err).Errorln("can't start listen connections to http api") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartListenConnections). + Errorln("System error: can't start listen connections to http API") os.Exit(1) } sigHandler.AddListener(commandsListener) for { connection, err := commandsListener.Accept() if err != nil { - log.WithError(err).Errorf("can't accept new connection") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorf("System error: can't accept new connection") continue } // unix socket and value == '@' if len(connection.RemoteAddr().String()) == 1 { - log.WithError(err).Errorf("new connection to http api: <%v>", connection.LocalAddr()) + log.Infof("Got new connection to http API: %v", connection.LocalAddr()) } else { - log.WithError(err).Errorf("new connection to http api: <%v>", connection.RemoteAddr()) + log.Infof("Got new connection to http API: %v", connection.RemoteAddr()) } go handleClientConnection(&commandsConfig, connection) } }() } - log.Infof("start listening %s", *connectionString) + + log.Infof("Start listening connection %s", *connectionString) for { connection, err := listener.Accept() if err != nil { - log.WithError(err).Errorln("can't accept new connection") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorln("System error: сan't accept new connection") os.Exit(1) } // unix socket and value == '@' if len(connection.RemoteAddr().String()) == 1 { - log.Infof("new connection to acraproxy: <%v>", connection.LocalAddr()) + log.Infof("Got new connection to acraproxy: %v", connection.LocalAddr()) } else { - log.Infof("new connection to acraproxy: <%v>", connection.RemoteAddr()) + log.Infof("Got new connection to acraproxy: %v", connection.RemoteAddr()) } go handleClientConnection(config, connection) } diff --git a/cmd/acraserver/acraserver b/cmd/acraserver/acraserver new file mode 100755 index 000000000..c711e9675 Binary files /dev/null and b/cmd/acraserver/acraserver differ diff --git a/cmd/acraserver/acraserver.go b/cmd/acraserver/acraserver.go index 73b0c436c..517c3178c 100644 --- a/cmd/acraserver/acraserver.go +++ b/cmd/acraserver/acraserver.go @@ -14,23 +14,52 @@ package main import ( + "crypto/tls" + "errors" "flag" "net/http" _ "net/http/pprof" "os" "syscall" + "time" "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" "github.com/cossacklabs/acra/utils" log "github.com/sirupsen/logrus" ) +var restartSignalsChannel chan os.Signal +var errorSignalChannel chan os.Signal +var err error +var authPath *string + +const ( + TEST_MODE = "true" +) + +var TestOnly = "false" + +const ( + DEFAULT_ACRASERVER_WAIT_TIMEOUT = 10 + GRACEFUL_ENV = "GRACEFUL_RESTART" + DESCRIPTOR_ACRA = 3 + DESCRIPTOR_API = 4 + SERVICE_NAME = "acraserver" +) + // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default -var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName("acraserver") +var DEFAULT_CONFIG_PATH = utils.GetConfigPathByName(SERVICE_NAME) +var ErrWaitTimeout = errors.New("timeout") func main() { + config := NewConfig() + loggingFormat := flag.String("logging_format", "plaintext", "Logging format: plaintext, json or CEF") + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + log.Infof("Starting service") + dbHost := flag.String("db_host", "", "Host to db") dbPort := flag.Int("db_port", 5432, "Port to db") @@ -45,18 +74,19 @@ func main() { serverId := flag.String("server_id", "acra_server", "Id that will be sent in secure session") - verbose := flag.Bool("v", false, "Log to stdout") + verbose := flag.Bool("v", false, "Log to stderr") flag.Bool("wholecell", true, "Acrastruct will stored in whole data cell") injectedcell := flag.Bool("injectedcell", false, "Acrastruct may be injected into any place of data cell") debug := flag.Bool("d", false, "Turn on debug logging") debugServer := flag.Bool("ds", false, "Turn on http debug server") + closeConnectionTimeout := flag.Int("close_connections_timeout", DEFAULT_ACRASERVER_WAIT_TIMEOUT, "Time that acraserver will wait (in seconds) on restart before closing all connections") stopOnPoison := flag.Bool("poisonshutdown", false, "Stop on detecting poison record") scriptOnPoison := flag.String("poisonscript", "", "Execute script on detecting poison record") withZone := flag.Bool("zonemode", false, "Turn on zone mode") - disableHTTPApi := flag.Bool("disable_http_api", false, "Disable http api") + enableHTTPApi := flag.Bool("enable_http_api", false, "Enable HTTP API") useTls := flag.Bool("tls", false, "Use tls to encrypt transport between acraserver and acraproxy/client") tlsKey := flag.String("tls_key", "", "Path to tls server key") @@ -67,13 +97,24 @@ func main() { clientId := flag.String("client_id", "", "Expected client id of acraproxy in mode without encryption") acraConnectionString := flag.String("connection_string", network.BuildConnectionString(cmd.DEFAULT_ACRA_CONNECTION_PROTOCOL, cmd.DEFAULT_ACRA_HOST, cmd.DEFAULT_ACRA_PORT, ""), "Connection string like tcp://x.x.x.x:yyyy or unix:///path/to/socket") acraAPIConnectionString := flag.String("connection_api_string", network.BuildConnectionString(cmd.DEFAULT_ACRA_CONNECTION_PROTOCOL, cmd.DEFAULT_ACRA_HOST, cmd.DEFAULT_ACRA_API_PORT, ""), "Connection string for api like tcp://x.x.x.x:yyyy or unix:///path/to/socket") + authPath = flag.String("auth_keys", cmd.DEFAULT_ACRA_AUTH_PATH, "Path to basic auth passwords. To add user, use: `./acra_genauth --set --user --pwd `") + + useMysql := flag.Bool("mysql", false, "Handle MySQL connections") + usePostgresql := flag.Bool("postgresql", false, "Handle Postgresql connections (default true)") + censorConfig := flag.String("censor_config", "", "Path to acracensor configuration file") + err := cmd.Parse(DEFAULT_CONFIG_PATH) if err != nil { - log.WithError(err).Errorln("can't parse args") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadServiceConfig). + Errorln("Can't parse args") os.Exit(1) } + // if log format was overridden + logging.CustomizeLogging(*loggingFormat, SERVICE_NAME) + + log.Infof("Validating service configuration") cmd.ValidateClientId(*serverId) if *host != cmd.DEFAULT_ACRA_HOST || *port != cmd.DEFAULT_ACRA_PORT { @@ -84,19 +125,36 @@ func main() { } if *debug { - cmd.SetLogLevel(cmd.LOG_DEBUG) + logging.SetLogLevel(logging.LOG_DEBUG) } else if *verbose { - cmd.SetLogLevel(cmd.LOG_VERBOSE) + logging.SetLogLevel(logging.LOG_VERBOSE) } else { - cmd.SetLogLevel(cmd.LOG_DISCARD) + logging.SetLogLevel(logging.LOG_DISCARD) } if *dbHost == "" { - log.Errorln("you must specify db_host") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorln("db_host is empty: you must specify db_host") flag.Usage() return } - config := NewConfig() + if err := config.SetMySQL(*useMysql); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorln("Can't set MySQL support") + os.Exit(1) + } + if err := config.SetPostgresql(*usePostgresql); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorWrongConfiguration). + Errorln("Can't set PostgreSQL support") + os.Exit(1) + } + + if err := config.SetCensor(*censorConfig); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCensorSetupError). + Errorln("Can't setup censor") + os.Exit(1) + } + // now it's stub as default values config.SetStopOnPoison(*stopOnPoison) config.SetScriptOnPoison(*scriptOnPoison) @@ -113,69 +171,203 @@ func main() { config.SetTLSServerCertPath(*tlsCert) config.SetTLSServerKeyPath(*tlsKey) config.SetWholeMatch(!(*injectedcell)) + config.SetEnableHTTPApi(*enableHTTPApi) + config.SetConfigPath(DEFAULT_CONFIG_PATH) + config.SetDebug(*debug) + + if *hexFormat || !*escapeFormat { config.SetByteaFormat(HEX_BYTEA_FORMAT) } else { config.SetByteaFormat(ESCAPE_BYTEA_FORMAT) } - keyStore, err := keystore.NewFilesystemKeyStore(*keysDir) + log.Infof("Initialising keystore") + masterKey, err := keystore.GetMasterKeyFromEnvironment() if err != nil { - log.Errorln("can't initialize keystore") + log.WithError(err).Errorln("can't load master key") os.Exit(1) } - if *useTls { - log.Println("use TLS transport wrapper") - tlsConfig, err := network.NewTLSConfig(*tlsSNI, *tlsCA, *tlsKey, *tlsCert) + scellEncryptor, err := keystore.NewSCellKeyEncryptor(masterKey) + if err != nil { + log.WithError(err).Errorln("can't init scell encryptor") + os.Exit(1) + } + keyStore, err := keystore.NewFilesystemKeyStore(*keysDir, scellEncryptor) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantInitKeyStore). + Errorln("Can't initialise keystore") + os.Exit(1) + } + var tlsConfig *tls.Config + if *useTls || *tlsKey != "" { + tlsConfig, err = network.NewTLSConfig(*tlsSNI, *tlsCA, *tlsKey, *tlsCert) if err != nil { - log.WithError(err).Errorln("can't get config for TLS") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't get config for TLS") os.Exit(1) } + // need for testing with mysql docker container that always generate new certificates + if TestOnly == TEST_MODE { + tlsConfig.InsecureSkipVerify = true + tlsConfig.ClientAuth = tls.NoClientCert + log.Warningln("Skip verifying TLS certificate, use for tests only!") + } + } + config.SetTLSConfig(tlsConfig) + if *useTls { + log.Println("Using TLS transport wrapper") config.ConnectionWrapper, err = network.NewTLSConnectionWrapper([]byte(*clientId), tlsConfig) if err != nil { - log.Errorln("can't initialize tls connection wrapper") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't initialise TLS connection wrapper") os.Exit(1) } } else if *noEncryption { if *clientId == "" && !*withZone { - log.Errorln("without zone mode and without encryption you must set which will be used to connect from acraproxy to acraserver") + log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: without zone mode and without encryption you must set which will be used to connect from acraproxy to acraserver") os.Exit(1) } - log.Println("use raw transport wrapper") + log.Infof("Selecting transport: use raw transport wrapper") config.ConnectionWrapper = &network.RawConnectionWrapper{ClientId: []byte(*clientId)} } else { - log.Println("use Secure Session transport wrapper") + log.Infof("Selecting transport: use Secure Session transport wrapper") config.ConnectionWrapper, err = network.NewSecureSessionConnectionWrapper(keyStore) if err != nil { - log.Errorln("can't initialize secure session connection wrapper") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTransportConfiguration). + Errorln("Configuration error: can't initialize secure session connection wrapper") os.Exit(1) } } - server, err := NewServer(config, keyStore) + log.Debugf("Registering process signal handlers") + sigHandlerSIGTERM, err := cmd.NewSignalHandler([]os.Signal{os.Interrupt, syscall.SIGTERM}) + errorSignalChannel = sigHandlerSIGTERM.GetChannel() if err != nil { - panic(err) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantRegisterSignalHandler). + Errorln("System error: can't register SIGTERM handler") + os.Exit(1) } - sigHandler, err := cmd.NewSignalHandler([]os.Signal{os.Interrupt, syscall.SIGTERM}) + sigHandlerSIGHUP, err := cmd.NewSignalHandler([]os.Signal{syscall.SIGHUP}) + restartSignalsChannel = sigHandlerSIGHUP.GetChannel() if err != nil { - log.WithError(err).Errorln("can't register SIGINT handler") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantRegisterSignalHandler). + Errorln("System error: can't register SIGHUP handler") os.Exit(1) } - go sigHandler.Register() - sigHandler.AddCallback(func() { server.Close() }) + + var server *SServer + server, err = NewServer(config, keyStore, errorSignalChannel, restartSignalsChannel) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartService). + Errorln("System error: can't start %s", SERVICE_NAME) + panic(err) + } + + if os.Getenv(GRACEFUL_ENV) == "true" { + server.fddACRA = DESCRIPTOR_ACRA + server.fdAPI = DESCRIPTOR_API + log.Debugf("Will be using GRACEFUL_RESTART if configured from WebUI") + } if *debugServer { //start http server for pprof + debugServerAddress := "127.0.0.1:6060" + log.Debugf("Starting Debug server on %s", debugServerAddress) go func() { - err := http.ListenAndServe("127.0.0.1:6060", nil) + err := http.ListenAndServe(debugServerAddress, nil) if err != nil { - log.WithError(err).Errorln("error from debug server") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartService). + Errorln("System error: got error from Debug Server") } }() } - if *withZone && !*disableHTTPApi { - go server.StartCommands() + + go sigHandlerSIGTERM.Register() + sigHandlerSIGTERM.AddCallback(func() { + log.Infof("Received incoming SIGTERM or SIGINT signal") + log.Debugf("Stop accepting new connections, waiting until current connections close") + // Stop accepting new connections + server.StopListeners() + // Wait a maximum of N seconds for existing connections to finish + err := server.WaitWithTimeout(time.Duration(*closeConnectionTimeout) * time.Second) + if err == ErrWaitTimeout { + log.Warningf("Server shutdown Timeout: %d active connections will be cut", server.ConnectionsCounter()) + server.Close() + os.Exit(1) + } + server.Close() + log.Infof("Server graceful shutdown completed, bye PID: %v", os.Getpid()) + os.Exit(0) + }) + + sigHandlerSIGHUP.AddCallback(func() { + log.Infof("Received incoming SIGHUP signal") + log.Debugf("Stop accepting new connections, waiting until current connections close") + + // Stop accepting requests + server.StopListeners() + + // Get socket file descriptor to pass it to fork + var fdACRA, fdAPI uintptr + fdACRA, err = network.ListenerFileDescriptor(server.listenerACRA) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetFileDescriptor). + Fatalln("System error: failed to get acra-socket file descriptor:", err) + } + if *withZone || *enableHTTPApi { + fdAPI, err = network.ListenerFileDescriptor(server.listenerAPI) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetFileDescriptor). + Fatalln("System error: failed to get api-socket file descriptor:", err) + } + } + + // Set env flag for forked process + os.Setenv(GRACEFUL_ENV, "true") + execSpec := &syscall.ProcAttr{ + Env: os.Environ(), + Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), fdACRA, fdAPI}, + } + + log.Debugf("Forking new process of %s", SERVICE_NAME) + + // Fork new process + var fork, err = syscall.ForkExec(os.Args[0], os.Args, execSpec) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantForkProcess). + Fatalln("System error: failed to fork new process", err) + } + log.Infof("%s process forked to PID: %v", SERVICE_NAME, fork) + + // Wait a maximum of N seconds for existing connections to finish + err = server.WaitWithTimeout(time.Duration(*closeConnectionTimeout) * time.Second) + if err == ErrWaitTimeout { + log.Warningf("Server shutdown Timeout: %d active connections will be cut", server.ConnectionsCounter()) + os.Exit(0) + } + log.Infof("Server graceful restart completed, bye PID: %v", os.Getpid()) + + // Stop the old server, all the connections have been closed and the new one is running + os.Exit(0) + }) + + log.Infof("Start listening to connections. Current PID: %v", os.Getpid()) + + if os.Getenv(GRACEFUL_ENV) == "true" { + go server.StartFromFileDescriptor(DESCRIPTOR_ACRA) + if *withZone || *enableHTTPApi { + go server.StartCommandsFromFileDescriptor(DESCRIPTOR_API) + } + } else { + go server.Start() + if *withZone || *enableHTTPApi { + go server.StartCommands() + } } - server.Start() + + // todo: any reason why it's so far from adding callback? + sigHandlerSIGHUP.Register() } diff --git a/cmd/acraserver/auth.go b/cmd/acraserver/auth.go new file mode 100644 index 000000000..f6692028e --- /dev/null +++ b/cmd/acraserver/auth.go @@ -0,0 +1,44 @@ +// Copyright 2018, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "github.com/cossacklabs/acra/utils" +) + +var ErrGetAuthDataFromFile = errors.New(fmt.Sprintf("No auth config [%v]", authPath)) + +func getAuthDataFromFile(authPath string) (data []byte, err error) { + configPath, err := utils.AbsPath(authPath) + if err != nil { + return nil, err + } + exists, err := utils.FileExists(configPath) + if err != nil { + return nil, err + } + if exists { + fileContent, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + data = fileContent + return data, nil + } + return nil, ErrGetAuthDataFromFile +} diff --git a/cmd/acraserver/client_commands_session.go b/cmd/acraserver/client_commands_session.go index f8abaa118..0e5595c41 100644 --- a/cmd/acraserver/client_commands_session.go +++ b/cmd/acraserver/client_commands_session.go @@ -15,21 +15,32 @@ package main import ( "bufio" + "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" "net" "net/http" + "encoding/json" "errors" + "flag" + "fmt" + "github.com/cossacklabs/acra/cmd" "github.com/cossacklabs/acra/keystore" "github.com/cossacklabs/acra/utils" "github.com/cossacklabs/acra/zone" + "github.com/cossacklabs/themis/gothemis/cell" "github.com/cossacklabs/themis/gothemis/keys" - "fmt" - "encoding/json" + "syscall" +) + +const ( + RESPONSE_500_ERROR = "HTTP/1.1 500 Server error\r\n\r\n\r\n\r\n" ) type ClientCommandsSession struct { ClientSession + Server *SServer + keystore keystore.KeyStore } func NewClientCommandsSession(keystorage keystore.KeyStore, config *Config, connection net.Conn) (*ClientCommandsSession, error) { @@ -37,7 +48,7 @@ func NewClientCommandsSession(keystorage keystore.KeyStore, config *Config, conn if err != nil { return nil, err } - return &ClientCommandsSession{ClientSession: *clientSession}, nil + return &ClientCommandsSession{ClientSession: *clientSession, keystore: keystorage}, nil } @@ -46,62 +57,115 @@ func (clientSession *ClientCommandsSession) ConnectToDb() error { } func (clientSession *ClientCommandsSession) close() { - log.Debugln("close acraproxy connection") + log.Debugln("Close acraproxy connection") err := clientSession.connection.Close() if err != nil { - log.Warningf("%v", utils.ErrorMessage("error with closing connection to acraproxy", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection). + Errorln("Error during closing connection to acraproxy") } - log.Debugln("all connections closed") + log.Debugln("All connections closed") } func (clientSession *ClientCommandsSession) HandleSession() { reader := bufio.NewReader(clientSession.connection) req, err := http.ReadRequest(reader) + // req = clientSession.connection.Write(*http.ResponseWriter) if err != nil { - log.Warningf("%v", utils.ErrorMessage("error reading command request from proxy", err)) + + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral). + Warningln("Got new command request, but can't read it") clientSession.close() return } response := "HTTP/1.1 404 Not Found\r\n\r\nincorrect request\r\n\r\n" - log.Debugln(req.URL.Path) + log.Debugf("Incoming API request to %v", req.URL.Path) switch req.URL.Path { case "/getNewZone": + log.Debugln("Got /getNewZone request") id, publicKey, err := clientSession.keystorage.GenerateZoneKey() if err == nil { zoneData, err := zone.ZoneDataToJson(id, &keys.PublicKey{Value: publicKey}) if err == nil { + log.Debugln("Handled request correctly") response = fmt.Sprintf("HTTP/1.1 200 OK Found\r\n\r\n%s\r\n\r\n", string(zoneData)) } } case "/resetKeyStorage": - log.Info("clear key storage cache") + log.Debugln("Got /resetKeyStorage request") clientSession.keystorage.Reset() response = "HTTP/1.1 200 OK Found\r\n\r\n" + log.Debugln("Cleared key storage cache") + case "/loadAuthData": + response = RESPONSE_500_ERROR + key, err := clientSession.keystore.GetAuthKey(false) + if err != nil { + log.WithError(err).Error("loadAuthData: keystore.GetAuthKey()") + response = RESPONSE_500_ERROR + break + } + authDataCrypted, err := getAuthDataFromFile(*authPath) + if err != nil { + log.Warningf("%v\n", utils.ErrorMessage("loadAuthData: no auth data", err)) + response = RESPONSE_500_ERROR + break + } + SecureCell := cell.New(key, cell.CELL_MODE_SEAL) + authData, err := SecureCell.Unprotect(authDataCrypted, nil, nil) + if err != nil { + log.WithError(err).Error("loadAuthData: SecureCell.Unprotect") + + break + } + response = fmt.Sprintf("HTTP/1.1 200 OK Found\r\n\r\n%s\r\n\r\n", authData) case "/getConfig": + log.Debugln("Got /getConfig request") jsonOutput, err := clientSession.config.ToJson() if err != nil { - log.Warningf("%v\n", utils.ErrorMessage("can't convert config to JSON", err)) - response = "HTTP/1.1 500 Server error\r\n\r\n\r\n\r\n" + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral). + Warningln("Can't convert config to JSON") + response = RESPONSE_500_ERROR } else { + log.Debugln("Handled request correctly") log.Debugln(string(jsonOutput)) response = fmt.Sprintf("HTTP/1.1 200 OK Found\r\n\r\n%s\r\n\r\n", string(jsonOutput)) } case "/setConfig": + log.Debugln("Got /setConfig request") decoder := json.NewDecoder(req.Body) var configFromUI UIEditableConfig err := decoder.Decode(&configFromUI) if err != nil { - log.Warningf("%v\n", utils.ErrorMessage("can't convert config from incoming", err)) - response = "HTTP/1.1 500 Server error\r\n\r\n\r\n\r\n" + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral). + Warningln("Can't convert config from incoming") + response = RESPONSE_500_ERROR + return + } + // set config values + flag.Set("db_host", configFromUI.DbHost) + flag.Set("db_port", fmt.Sprintf("%v", configFromUI.DbPort)) + flag.Set("commands_port", fmt.Sprintf("%v", configFromUI.ProxyCommandsPort)) + flag.Set("d", fmt.Sprintf("%v", configFromUI.Debug)) + flag.Set("poisonscript", fmt.Sprintf("%v", configFromUI.ScriptOnPoison)) + flag.Set("poisonshutdown", fmt.Sprintf("%v", configFromUI.StopOnPoison)) + flag.Set("zonemode", fmt.Sprintf("%v", configFromUI.WithZone)) + + err = cmd.DumpConfig(clientSession.Server.config.GetConfigPath(), false) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantDumpConfig). + Errorln("DumpConfig failed") + response = RESPONSE_500_ERROR + return + } - log.Debugln(configFromUI) + log.Debugln("Handled request correctly, restarting server") + clientSession.Server.restartSignalsChannel <- syscall.SIGHUP } _, err = clientSession.connection.Write([]byte(response)) if err != nil { - log.Warningf("%v", utils.ErrorMessage("can't send data with secure session to acraproxy", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral).Errorln("Can't send data with secure session to acraproxy") return } clientSession.close() diff --git a/cmd/acraserver/client_session.go b/cmd/acraserver/client_session.go index 9450abc1f..8d25d4910 100644 --- a/cmd/acraserver/client_session.go +++ b/cmd/acraserver/client_session.go @@ -15,15 +15,18 @@ package main import ( "fmt" + "net" + + "github.com/cossacklabs/acra/decryptor/mysql" + "github.com/cossacklabs/acra/decryptor/postgresql" "github.com/cossacklabs/acra/network" log "github.com/sirupsen/logrus" - "net" + + "io" "github.com/cossacklabs/acra/decryptor/base" - "github.com/cossacklabs/acra/decryptor/postgresql" "github.com/cossacklabs/acra/keystore" - "github.com/cossacklabs/acra/utils" - "io" + "github.com/cossacklabs/acra/logging" ) type ClientSession struct { @@ -31,6 +34,7 @@ type ClientSession struct { keystorage keystore.KeyStore connection net.Conn connectionToDb net.Conn + Server *SServer } func NewClientSession(keystorage keystore.KeyStore, config *Config, connection net.Conn) (*ClientSession, error) { @@ -47,48 +51,59 @@ func (clientSession *ClientSession) ConnectToDb() error { } func (clientSession *ClientSession) close() { - log.Debugln("close acraproxy connection") + log.Debugln("Close acraproxy connection") err := clientSession.connection.Close() if err != nil { - log.Warningf("%v", utils.ErrorMessage("error with closing connection to acraproxy", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnectionToService). + Errorln("Error with closing connection to acraproxy") } - log.Debugln("close db connection") + log.Debugln("Close db connection") err = clientSession.connectionToDb.Close() if err != nil { - log.Warningf("%v", utils.ErrorMessage("error with closing connection to db", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnectionDB). + Errorln("Error with closing connection to db") } - log.Debugln("all connections closed") + log.Debugln("All connections closed") } /* proxy connections from client to db and decrypt responses from db to client if any error occurred than end processing */ -func (clientSession *ClientSession) HandleSecureSession(decryptorImpl base.Decryptor) { +func (clientSession *ClientSession) HandleClientConnection(decryptorImpl base.Decryptor) { + log.Infof("Handle client's connection") innerErrorChannel := make(chan error, 2) + log.Debugf("Connecting to db") err := clientSession.ConnectToDb() if err != nil { - log.WithError(err).Errorln("can't connect to db") - log.Debugln("close connection with acraproxy") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantConnectToDB). + Errorln("Can't connect to db") + + log.Debugln("Close connection with acraproxy") err = clientSession.connection.Close() if err != nil { - log.Warningf("%v", utils.ErrorMessage("error with closing connection to acraproxy", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnectionToService). + Errorln("Error with closing connection to acraproxy") } return } - pgDecryptorConfig, err := postgresql.NewPgDecryptorConfig(clientSession.config.GetTLSServerKeyPath(), clientSession.config.GetTLSServerCertPath()) - if err != nil { - log.WithError(err).Errorln("can't initialize config for postgresql decryptor") - err = clientSession.connection.Close() + + if clientSession.config.UseMySQL() { + log.Debugln("MySQL connection") + handler, err := mysql.NewMysqlHandler(decryptorImpl, clientSession.connectionToDb, clientSession.connection, clientSession.config.GetTLSConfig(), clientSession.config.censor) if err != nil { - log.Warningf("%v", utils.ErrorMessage("error with closing connection to acraproxy", err)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantInitDecryptor). + Errorln("Can't initialize mysql handler") + return } - return + go handler.ClientToDbProxy(innerErrorChannel) + go handler.DbToClientProxy(innerErrorChannel) + } else { + log.Debugln("PostgreSQL connection") + go network.Proxy(clientSession.connection, clientSession.connectionToDb, innerErrorChannel) + go postgresql.PgDecryptStream(decryptorImpl, clientSession.config.GetTLSConfig(), clientSession.connectionToDb, clientSession.connection, innerErrorChannel) } - - go network.Proxy(clientSession.connection, clientSession.connectionToDb, innerErrorChannel) - go postgresql.PgDecryptStream(decryptorImpl, pgDecryptorConfig, clientSession.connectionToDb, clientSession.connection, innerErrorChannel) for { err = <-innerErrorChannel @@ -96,17 +111,25 @@ func (clientSession *ClientSession) HandleSecureSession(decryptorImpl base.Decry log.Debugln("EOF connection closed") } else if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { - log.Debugln("network timeout") - continue + log.Debugln("Network timeout") + if clientSession.config.UseMySQL() { + break + } else { + // in postgresql mode timeout used to stop listening connection in background goroutine + // and it's normal behaviour + continue + } } - log.WithError(netErr).Errorln("network error") + log.WithError(netErr).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantHandleSecureSession). + Errorln("Network error") } else if opErr, ok := err.(*net.OpError); ok { - log.WithError(opErr).Errorln("network error") + log.WithError(opErr).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantHandleSecureSession).Errorln("Network error") } else { - log.WithError(err).Errorln("unexpected error") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantHandleSecureSession).Errorln("Unexpected error") } break } + log.Infof("Closing client's connection") clientSession.close() // wait second error from closed second connection <-innerErrorChannel diff --git a/cmd/acraserver/config.go b/cmd/acraserver/config.go index aeb0fc2cb..f9b189572 100644 --- a/cmd/acraserver/config.go +++ b/cmd/acraserver/config.go @@ -14,9 +14,13 @@ package main import ( - "errors" + "crypto/tls" "encoding/json" + "errors" + + "github.com/cossacklabs/acra/acracensor" "github.com/cossacklabs/acra/network" + "io/ioutil" ) const ( @@ -35,6 +39,7 @@ type Config struct { scriptOnPoison string stopOnPoison bool withZone bool + withAPI bool wholeMatch bool serverId []byte acraConnectionString string @@ -42,11 +47,15 @@ type Config struct { tlsServerKeyPath string tlsServerCertPath string ConnectionWrapper network.ConnectionWrapper + mysql bool + postgresql bool + configPath string + debug bool + censor acracensor.AcracensorInterface + tlsConfig *tls.Config } type UIEditableConfig struct { - ProxyHost string `json:"host"` - ProxyPort int `json:"port"` DbHost string `json:"db_host"` DbPort int `json:"db_port"` ProxyCommandsPort int `json:"commands_port"` @@ -57,7 +66,57 @@ type UIEditableConfig struct { } func NewConfig() *Config { - return &Config{withZone: false, stopOnPoison: false, wholeMatch: true} + return &Config{withZone: false, stopOnPoison: false, wholeMatch: true, mysql: false, postgresql: false} +} + +var ErrTwoDBSetup = errors.New("only one db supported at one time") + +func (config *Config) SetCensor(censorConfigPath string) error { + censor := &acracensor.AcraCensor{} + config.censor = censor + //skip if flag not specified + if censorConfigPath == "" { + return nil + } + configuration, err := ioutil.ReadFile(censorConfigPath) + if err != nil { + return err + } + err = censor.LoadConfiguration(configuration) + if err != nil { + return err + } + return nil +} +func (config *Config) GetCensor() acracensor.AcracensorInterface { + return config.censor +} + +func (config *Config) SetMySQL(useMySQL bool) error { + if config.postgresql && useMySQL { + return ErrTwoDBSetup + } + config.mysql = useMySQL + return nil +} +func (config *Config) UseMySQL() bool { + return config.mysql +} + +func (config *Config) UsePostgreSQL() bool { + // default true if two settings is false + if !(config.mysql || config.postgresql) { + return true + } + return config.postgresql +} + +func (config *Config) SetPostgresql(usePostgresql bool) error { + if config.mysql && usePostgresql { + return ErrTwoDBSetup + } + config.postgresql = usePostgresql + return nil } func (config *Config) GetTLSServerKeyPath() string { return config.tlsServerKeyPath @@ -89,12 +148,24 @@ func (config *Config) SetStopOnPoison(stop bool) { func (config *Config) GetStopOnPoison() bool { return config.stopOnPoison } +func (config *Config) SetDebug(value bool) { + config.debug = value +} +func (config *Config) GetDebug() bool { + return config.debug +} func (config *Config) GetWithZone() bool { return config.withZone } func (config *Config) SetWithZone(wz bool) { config.withZone = wz } +func (config *Config) SetEnableHTTPApi(api bool) { + config.withAPI = api +} +func (config *Config) GetEnableHTTPApi() bool { + return config.withAPI +} func (config *Config) GetProxyHost() string { return config.proxyHost } @@ -160,14 +231,19 @@ func (config *Config) GetWholeMatch() bool { func (config *Config) SetWholeMatch(value bool) { config.wholeMatch = value } +func (config *Config) GetConfigPath() string { + return config.configPath +} +func (config *Config) SetConfigPath(value string) { + config.configPath = value +} func (config *Config) ToJson() ([]byte, error) { var s UIEditableConfig - s.ProxyHost = config.GetProxyHost() - s.ProxyPort = config.GetProxyPort() s.DbHost = config.GetDBHost() s.DbPort = config.GetDBPort() s.ProxyCommandsPort = config.GetProxyCommandsPort() + s.Debug = config.GetDebug() s.ScriptOnPoison = config.GetScriptOnPoison() s.StopOnPoison = config.GetStopOnPoison() s.WithZone = config.GetWithZone() @@ -182,3 +258,10 @@ func (config *Config) GetAcraConnectionString() string { func (config *Config) GetAcraAPIConnectionString() string { return config.acraAPIConnectionString } + +func (config *Config) SetTLSConfig(tlsConfig *tls.Config) { + config.tlsConfig = tlsConfig +} +func (config *Config) GetTLSConfig() *tls.Config { + return config.tlsConfig +} diff --git a/cmd/acraserver/listener.go b/cmd/acraserver/listener.go index c64cbda07..e3fbff156 100644 --- a/cmd/acraserver/listener.go +++ b/cmd/acraserver/listener.go @@ -15,35 +15,81 @@ package main import ( "net" - - "github.com/cossacklabs/acra/network" - log "github.com/sirupsen/logrus" + url_ "net/url" + "os" + "syscall" + "time" "github.com/cossacklabs/acra/decryptor/base" + "github.com/cossacklabs/acra/decryptor/mysql" pg "github.com/cossacklabs/acra/decryptor/postgresql" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" + "github.com/cossacklabs/acra/network" "github.com/cossacklabs/acra/zone" + log "github.com/sirupsen/logrus" ) type SServer struct { - config *Config - keystorage keystore.KeyStore - listeners []net.Listener + config *Config + keystorage keystore.KeyStore + listenerACRA net.Listener + listenerAPI net.Listener + fddACRA uintptr + fdAPI uintptr + cmACRA *network.ConnectionManager + cmAPI *network.ConnectionManager + listeners []net.Listener + errorSignalChannel chan os.Signal + restartSignalsChannel chan os.Signal } -func NewServer(config *Config, keystorage keystore.KeyStore) (server *SServer, err error) { - return &SServer{config: config, keystorage: keystorage}, nil +func NewServer(config *Config, keystorage keystore.KeyStore, errorChan chan os.Signal, restarChan chan os.Signal) (server *SServer, err error) { + return &SServer{ + config: config, + keystorage: keystorage, + cmACRA: network.NewConnectionManager(), + cmAPI: network.NewConnectionManager(), + errorSignalChannel: errorChan, + restartSignalsChannel: restarChan, + }, nil } // Close all listeners and return first error -func (server *SServer) Close() error { +func (server *SServer) Close() { + log.Debugln("Closing server listeners..") var err error for _, listener := range server.listeners { - if err_ := listener.Close(); err_ != nil && err == nil { - err = err_ + switch listener.(type) { + case *net.TCPListener: + err = listener.(*net.TCPListener).Close() + if err != nil { + log.WithError(err).Infoln("TCPListener.Close()") + continue + } + case *net.UnixListener: + err = listener.(*net.UnixListener).Close() + if err != nil { + log.WithError(err).Infoln("UnixListener.Close()") + continue + } + // TODO: find better way to remove unixsocket file + url, err2 := url_.Parse(server.config.GetAcraConnectionString()) + if err2 != nil { + log.WithError(err2).Warningln("UnixListener.Close url_.Parse") + } + if _, err := os.Stat(url.Path); err == nil { + err3 := os.Remove(url.Path) + if err3 != nil { + log.WithError(err3).Warningf("UnixListener.Close file.Remove(%s)", url.Path) + } + } } } - return err + if err != nil { + log.WithError(err).Infoln("server.Close()") + } + log.Debugln("Closed server listeners") } func (server *SServer) addListener(listener net.Listener) { @@ -60,12 +106,12 @@ func (server *SServer) getDecryptor(clientId []byte) base.Decryptor { dataDecryptor = pg.NewPgEscapeDecryptor() matcherPool = zone.NewMatcherPool(zone.NewPgEscapeMatcherFactory()) } - decryptorImpl := pg.NewPgDecryptor(clientId, dataDecryptor) - decryptorImpl.SetWithZone(server.config.GetWithZone()) - decryptorImpl.SetWholeMatch(server.config.GetWholeMatch()) - decryptorImpl.SetKeyStore(server.keystorage) + pgDecryptorImpl := pg.NewPgDecryptor(clientId, dataDecryptor) + pgDecryptorImpl.SetWithZone(server.config.GetWithZone()) + pgDecryptorImpl.SetWholeMatch(server.config.GetWholeMatch()) + pgDecryptorImpl.SetKeyStore(server.keystorage) zoneMatcher := zone.NewZoneMatcher(matcherPool, server.keystorage) - decryptorImpl.SetZoneMatcher(zoneMatcher) + pgDecryptorImpl.SetZoneMatcher(zoneMatcher) poisonCallbackStorage := base.NewPoisonCallbackStorage() if server.config.GetScriptOnPoison() != "" { @@ -75,76 +121,214 @@ func (server *SServer) getDecryptor(clientId []byte) base.Decryptor { if server.config.GetStopOnPoison() { poisonCallbackStorage.AddCallback(&base.StopCallback{}) } - decryptorImpl.SetPoisonCallbackStorage(poisonCallbackStorage) - return decryptorImpl + pgDecryptorImpl.SetPoisonCallbackStorage(poisonCallbackStorage) + var decryptor base.Decryptor = pgDecryptorImpl + if server.config.UseMySQL() { + decryptor = mysql.NewMySQLDecryptor(pgDecryptorImpl, server.keystorage) + } + return decryptor } /* -handle new connection by iniailizing secure session, starting proxy request +handle new connection by initializing secure session, starting proxy request to db and decrypting responses from db */ func (server *SServer) handleConnection(connection net.Conn) { + log.Infof("Handle new connection") wrappedConnection, clientId, err := server.config.ConnectionWrapper.WrapServer(connection) if err != nil { - log.WithError(err).Println("can't wrap connection from acraproxy") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection). + Errorln("Can't wrap connection from acraproxy") if closeErr := connection.Close(); closeErr != nil { - log.WithError(closeErr).Println("can't close connection") + log.WithError(closeErr).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection). + Errorln("Can't close connection") } return } clientSession, err := NewClientSession(server.keystorage, server.config, connection) + clientSession.Server = server if err != nil { - log.WithError(err).Println("can't initialize client session") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantInitClientSession). + Errorln("Can't initialize client session") if closeErr := connection.Close(); closeErr != nil { - log.WithError(closeErr).Println("can't close connection") + log.WithError(closeErr).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection). + Errorln("Can't close connection") } return } clientSession.connection = wrappedConnection decryptor := server.getDecryptor(clientId) - clientSession.HandleSecureSession(decryptor) + clientSession.HandleClientConnection(decryptor) } // start listening connections from proxy func (server *SServer) Start() { - listener, err := network.Listen(server.config.GetAcraConnectionString()) + var connection net.Conn + var listener, err = network.Listen(server.config.GetAcraConnectionString()) if err != nil { - log.WithError(err).Errorln("can't start listen connections") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartListenConnections). + Errorln("Can't start listen connections") + server.errorSignalChannel <- syscall.SIGTERM return } - defer listener.Close() + server.listenerACRA = listener server.addListener(listener) - log.Infof("start listening %s", server.config.GetAcraConnectionString()) + + log.Infof("Start listening connection: %s", server.config.GetAcraConnectionString()) for { - connection, err := listener.Accept() + connection, err = listener.Accept() if err != nil { - log.WithError(err).Errorln("can't accept new connection") + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorConnectionDroppedByTimeout). + Errorln("Stop accepting new connections due net.Timeout") + return + } + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorf("Can't accept new connection (connection=%v)", connection) continue } // unix socket and value == '@' if len(connection.RemoteAddr().String()) == 1 { - log.Infof("new connection to acraserver: <%v>", connection.LocalAddr()) + log.Infof("Got new connection to acraserver: %v", connection.LocalAddr()) } else { - log.Infof("new connection to acraserver: <%v>", connection.RemoteAddr()) + log.Infof("Got new connection to acraserver: %v", connection.RemoteAddr()) } - go server.handleConnection(connection) + go func() { + server.cmACRA.Incr() + server.handleConnection(connection) + server.cmACRA.Done() + }() + } } +func (server *SServer) StartFromFileDescriptor(fd uintptr) { + var connection net.Conn + file := os.NewFile(fd, "/tmp/acraserver") + listenerFile, err := net.FileListener(file) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantOpenFileByDescriptor). + Errorln("System error: can't start listen for file descriptor") + server.errorSignalChannel <- syscall.SIGTERM + return + } + + listenerWithFileDescriptor, ok := listenerFile.(network.ListenerWithFileDescriptor) + if !ok { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorFileDescriptionIsNotValid). + Errorf("System error: file descriptor %d is not a valid socket", fd) + return + } + server.listenerACRA = listenerWithFileDescriptor + server.addListener(listenerFile) + + log.Infof("Start listening connection: %s", server.config.GetAcraConnectionString()) + for { + connection, err = listenerWithFileDescriptor.Accept() + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorConnectionDroppedByTimeout). + Errorf("Stop accepting new connections", connection) + return + } + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorf("Can't accept new connection (connection=%v)", connection) + continue + } + // unix socket and value == '@' + if len(connection.RemoteAddr().String()) == 1 { + log.Infof("Got new connection to acraserver: %v", connection.LocalAddr()) + } else { + log.Infof("Got new connection to acraserver: %v", connection.RemoteAddr()) + } + go func() { + server.cmACRA.Incr() + server.handleConnection(connection) + server.cmACRA.Done() + }() + } +} + +func (server *SServer) StopListeners() { + var ( + err error + tcpListener *net.TCPListener + ok bool + ) + log.Debugln("Stopping listeners") + if tcpListener, ok = server.listenerACRA.(*net.TCPListener); ok { + err = tcpListener.SetDeadline(time.Now()) + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStopListenConnections). + Errorln("Unable to SetDeadLine of acra-listener") + } + } + } else { + log.Warningln("Acra-interface assigment failed") + } + + if server.listenerAPI != nil { + if tcpListener, ok = server.listenerAPI.(*net.TCPListener); ok { + err = tcpListener.SetDeadline(time.Now()) + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStopListenConnections). + Errorln("Unable to SetDeadLine of API-listener") + } + } + } else { + log.Warningln("API-interface assigment failed") + } + } +} + +func (server *SServer) WaitConnections(duration time.Duration) { + log.Infof("Waiting for %v connections to complete", server.ConnectionsCounter()) + server.cmACRA.Wait() + if server.listenerAPI != nil { + server.cmAPI.Wait() + } +} + +func (server *SServer) WaitWithTimeout(duration time.Duration) error { + timeout := time.NewTimer(duration) + wait := make(chan struct{}) + go func() { + server.WaitConnections(duration) + wait <- struct{}{} + }() + + select { + case <-timeout.C: + return ErrWaitTimeout + case <-wait: + return nil + } +} + +func (server *SServer) ConnectionsCounter() int { + return server.cmACRA.Counter + server.cmAPI.Counter +} + /* -handle new connection by iniailizing secure session, starting proxy request +handle new connection by initializing secure session, starting proxy request to db and decrypting responses from db */ func (server *SServer) handleCommandsConnection(connection net.Conn) { + log.Infof("Handle commands connection") clientSession, err := NewClientCommandsSession(server.keystorage, server.config, connection) + clientSession.Server = server if err != nil { - log.WithError(err).Errorln("can't init session") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartConnection). + Errorln("Can't init API session") return } wrappedConnection, _, err := server.config.ConnectionWrapper.WrapServer(connection) if err != nil { - log.WithError(err).Errorln("can't wrap connection") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection). + Errorln("Can't wrap API connection") return } clientSession.connection = wrappedConnection @@ -153,26 +337,86 @@ func (server *SServer) handleCommandsConnection(connection net.Conn) { // start listening commands connections from proxy func (server *SServer) StartCommands() { - listener, err := network.Listen(server.config.GetAcraAPIConnectionString()) + var connection net.Conn + var listener, err = network.Listen(server.config.GetAcraAPIConnectionString()) if err != nil { - log.WithError(err).Errorln("can't start listen command connections") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartListenConnections). + Errorln("Can't start listen command API connections") + server.errorSignalChannel <- syscall.SIGTERM return } - defer listener.Close() + server.listenerAPI = listener server.addListener(listener) - log.Infof("start listening api %s", server.config.GetAcraAPIConnectionString()) + + log.Infof("Start listening API: %s", server.config.GetAcraAPIConnectionString()) + for { + connection, err = listener.Accept() + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorln("Stop accepting new connections", connection) + return + } + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorf("Can't accept new connection (connection=%v)", connection) + continue + } + // unix socket and value == '@' + if len(connection.RemoteAddr().String()) == 1 { + log.Infof("Got new connection to http API: %v", connection.LocalAddr()) + } else { + log.Infof("Got new connection to http API: %v", connection.RemoteAddr()) + } + go func() { + server.cmAPI.Incr() + server.handleCommandsConnection(connection) + server.cmAPI.Done() + }() + } +} + +func (server *SServer) StartCommandsFromFileDescriptor(fd uintptr) { + var connection net.Conn + file := os.NewFile(fd, "/tmp/acraserver_http_api") + listenerFile, err := net.FileListener(file) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantOpenFileByDescriptor). + Errorln("System error: can't start listen for file descriptor") + server.errorSignalChannel <- syscall.SIGTERM + return + } + listenerWithFileDescriptor, ok := listenerFile.(network.ListenerWithFileDescriptor) + if !ok { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorFileDescriptionIsNotValid). + Errorf("System error: file descriptor %d is not a valid socket", fd) + return + } + server.listenerAPI = listenerWithFileDescriptor + server.addListener(listenerWithFileDescriptor) + + log.Infof("Start listening API from file descriptor: %s", server.config.GetAcraAPIConnectionString()) for { - connection, err := listener.Accept() + connection, err = listenerWithFileDescriptor.Accept() if err != nil { - log.WithError(err).Errorln("can't accept new connection") + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorConnectionDroppedByTimeout). + Errorln("Stop accepting new connections", connection) + return + } + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections). + Errorf("System error: can't accept new connection (connection=%v)", connection) continue } // unix socket and value == '@' if len(connection.RemoteAddr().String()) == 1 { - log.Infof("new connection to http api: <%v>", connection.LocalAddr()) + log.Infof("Got new connection to http API: %v", connection.LocalAddr()) } else { - log.Infof("new connection to http api: <%v>", connection.RemoteAddr()) + log.Infof("Got new connection to http API: %v", connection.RemoteAddr()) } - go server.handleCommandsConnection(connection) + go func() { + server.cmAPI.Incr() + server.handleCommandsConnection(connection) + server.cmAPI.Done() + }() } } diff --git a/cmd/constants.go b/cmd/constants.go index 02e96a6fc..6ad450181 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -5,8 +5,17 @@ const ( DEFAULT_PROXY_API_PORT = 9191 DEFAULT_PROXY_CONNECTION_PROTOCOL = "tcp" DEFAULT_PROXY_HOST = "127.0.0.1" - DEFAULT_ACRA_HOST = "0.0.0.0" - DEFAULT_ACRA_PORT = 9393 - DEFAULT_ACRA_API_PORT = 9090 + DEFAULT_ACRA_HOST = "0.0.0.0" + DEFAULT_ACRA_PORT = 9393 + DEFAULT_ACRA_API_PORT = 9090 + DEFAULT_ACRA_AUTH_PATH = "configs/auth.keys" DEFAULT_ACRA_CONNECTION_PROTOCOL = "tcp" + DEFAULT_ACRA_CONFIGUI_HOST = "127.0.0.1" + DEFAULT_ACRA_CONFIGUI_PORT = 8000 + DEFAULT_ACRA_CONFIGUI_STATIC = "cmd/acra_configui/static" + DEFAULT_ACRA_CONFIGUI_AUTH_MODE = "auth_on" + ACRA_CONFIGUI_AUTH_ARGON2_LENGTH = 32 + ACRA_CONFIGUI_AUTH_ARGON2_MEMORY = 8 * 1024 + ACRA_CONFIGUI_AUTH_ARGON2_TIME = 3 + ACRA_CONFIGUI_AUTH_ARGON2_THREADS = 2 ) diff --git a/cmd/utils.go b/cmd/utils.go index 03cf846d2..6d922fa64 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -15,6 +15,11 @@ import ( "github.com/cossacklabs/acra/utils" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "time" + "math/rand" + "strings" + "encoding/base64" + "strconv" ) var ( @@ -22,12 +27,6 @@ var ( dumpconfig = flag_.Bool("dumpconfig", false, "dump config") ) -const ( - LOG_DEBUG = iota - LOG_VERBOSE - LOG_DISCARD -) - func init() { // override default usage message by ours flag_.CommandLine.Usage = PrintDefaults @@ -49,6 +48,10 @@ func (handler *SignalHandler) AddListener(listener net.Listener) { handler.listeners = append(handler.listeners, listener) } +func (handler *SignalHandler) GetChannel() chan os.Signal { + return handler.ch +} + func (handler *SignalHandler) AddCallback(callback SignalCallback) { handler.callbacks = append(handler.callbacks, callback) } @@ -141,16 +144,54 @@ func PrintDefaults() { }) } -func GenerateYaml(output io.Writer) { +func GenerateYaml(output io.Writer, useDefault bool) { flag_.CommandLine.VisitAll(func(flag *flag_.Flag) { - s := fmt.Sprintf("# %v\n%v: %v\n", flag.Usage, flag.Name, flag.DefValue) + var s string + if useDefault { + s = fmt.Sprintf("# %v\n%v: %v\n", flag.Usage, flag.Name, flag.DefValue) + } else { + s = fmt.Sprintf("# %v\n%v: %v\n", flag.Usage, flag.Name, flag.Value) + } fmt.Fprint(output, s, "\n") }) } +func DumpConfig(configPath string, useDefault bool) error { + var absPath string + var err error + + if *config == "" { + absPath, err = utils.AbsPath(configPath) + if err != nil { + return err + } + } else { + absPath, err = utils.AbsPath(*config) + if err != nil { + return err + } + } + + dirPath := filepath.Dir(absPath) + err = os.MkdirAll(dirPath, 0744) + if err != nil { + return err + } + + file, err := os.Create(absPath) + if err != nil { + return err + } + defer file.Close() + + GenerateYaml(file, useDefault) + log.Infof("Config dumped to %s", configPath) + return nil +} + func Parse(configPath string) error { /*load from yaml config and cli. if dumpconfig option pass than generate config and exit*/ - + log.Debugf("Parsing config from path %v", configPath) // first parse using bultin flag err := flag_.CommandLine.Parse(os.Args[1:]) if err != nil { @@ -163,6 +204,7 @@ func Parse(configPath string) error { var args []string // parse yaml and add params that wasn't passed from cli if configPath != "" { + configPath, err := utils.AbsPath(configPath) if err != nil { return err @@ -200,50 +242,68 @@ func Parse(configPath string) error { } } // set options from config that wasn't set by cli + log.Debugln(args) err = flag_.CommandLine.Parse(args) if err != nil { return err } if *dumpconfig { - var absPath string - if *config == "" { - absPath, err = utils.AbsPath(configPath) - if err != nil { - return err - } - } else { - absPath, err = utils.AbsPath(*config) - if err != nil { - return err - } - } - - dirPath := filepath.Dir(absPath) - err = os.MkdirAll(dirPath, 0744) - if err != nil { - return err - } - - file, err := os.Create(absPath) - if err != nil { - return err - } - defer file.Close() - - GenerateYaml(file) + DumpConfig(configPath, true) os.Exit(0) } return nil } -func SetLogLevel(level int) { - if level == LOG_DEBUG { - log.SetLevel(log.DebugLevel) - } else if level == LOG_VERBOSE { - log.SetLevel(log.InfoLevel) - } else if level == LOG_DISCARD { - log.SetLevel(log.WarnLevel) - } else { - panic(fmt.Sprintf("Incorrect log level - %v", level)) +type Argon2Params struct { + Time uint32 + Memory uint32 + Threads uint8 + Length uint32 +} + +type UserAuth struct { + Salt string + Argon2Params + Hash []byte +} + +func (auth UserAuth) UserAuthString(userDataDelimiter string, paramsDelimiter string) (string) { + var userData []string + var argon2P []string + argon2P = append(argon2P, strconv.FormatUint(uint64(auth.Argon2Params.Time), 10)) + argon2P = append(argon2P, strconv.FormatUint(uint64(auth.Argon2Params.Memory), 10)) + argon2P = append(argon2P, strconv.FormatUint(uint64(auth.Argon2Params.Threads), 10)) + argon2P = append(argon2P, strconv.FormatUint(uint64(auth.Argon2Params.Length), 10)) + hash := base64.StdEncoding.EncodeToString(auth.Hash) + userData = append(userData, auth.Salt) + userData = append(userData, strings.Join(argon2P, paramsDelimiter)) + userData = append(userData, hash) + return strings.Join(userData, userDataDelimiter) +} + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = randSrc.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- } + + return string(b) } diff --git a/cmd/utils_argon2.go b/cmd/utils_argon2.go new file mode 100644 index 000000000..1c216efbf --- /dev/null +++ b/cmd/utils_argon2.go @@ -0,0 +1,38 @@ +// Copyright 2018, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import "golang.org/x/crypto/argon2" + +func InitArgon2Params() (Argon2Params) { + var p Argon2Params + p.Time = uint32(ACRA_CONFIGUI_AUTH_ARGON2_TIME) + p.Memory = uint32(ACRA_CONFIGUI_AUTH_ARGON2_MEMORY) + p.Threads = uint8(ACRA_CONFIGUI_AUTH_ARGON2_THREADS) + p.Length = uint32(ACRA_CONFIGUI_AUTH_ARGON2_LENGTH) + return p +} + +func HashArgon2(password string, salt string, p Argon2Params) (hash []byte, err error) { + passwordBytes := argon2.IDKey([]byte(password), []byte(salt), + p.Time, + p.Memory, + p.Threads, + p.Length) + if err != nil { + return + } + return passwordBytes, nil +} diff --git a/configs/acra_censor.example.yaml b/configs/acra_censor.example.yaml new file mode 100644 index 000000000..5682405c1 --- /dev/null +++ b/configs/acra_censor.example.yaml @@ -0,0 +1,15 @@ +handlers: + - handler: logger + filepath: censor_log + - handler: blacklist + queries: + - INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang'); + - SELECT AVG(Price) FROM Products; + tables: + - EMPLOYEE_TBL + - Customers + rules: + - SELECT * FROM EMPLOYEE WHERE CITY='Seattle'; + - handler: whitelist + tables: + - EMPLOYEE \ No newline at end of file diff --git a/configs/acra_configui.yaml b/configs/acra_configui.yaml new file mode 100644 index 000000000..df4f7206c --- /dev/null +++ b/configs/acra_configui.yaml @@ -0,0 +1,30 @@ +# Host for Acraserver HTTP endpoint or proxy +acra_host: localhost + +# Port for Acraserver HTTP endpoint or proxy +acra_port: 9191 + +# Mode for basic auth. Possible values: auth_on|auth_off_local|auth_off +auth_mode: auth_on + +# path to config +config: + +# Turn on debug logging +d: false + +# dump config +dumpconfig: false + +# Host for configUI HTTP endpoint +host: 127.0.0.1 + +# Logging format: plaintext, json or CEF +logging_format: plaintext + +# Port for configUI HTTP endpoint +port: 8000 + +# Path to static content +static_path: cmd/acra_configui/static + diff --git a/configs/acra_genkeys.yaml b/configs/acra_genkeys.yaml index be9a25a14..a2f4dd7d5 100644 --- a/configs/acra_genkeys.yaml +++ b/configs/acra_genkeys.yaml @@ -1,9 +1,12 @@ -# Create keypair only for acraproxy +# Create keypair for acraproxy only acraproxy: false -# Create keypair only for acraserver +# Create keypair for acraserver only acraserver: false +# Create symmetric key for acra_configui's basic auth db +basicauth: false + # Client id client_id: client @@ -13,9 +16,15 @@ config: # dump config dumpconfig: false +# Generate new random master key and save to file +master_key: + # Folder where will be saved keys output: .acrakeys +# Folder where will be saved public key +output_public: .acrakeys + # Create keypair for data encryption/decryption storage: false diff --git a/configs/acra_rollback.yaml b/configs/acra_rollback.yaml index 804b599af..eca8197c9 100644 --- a/configs/acra_rollback.yaml +++ b/configs/acra_rollback.yaml @@ -16,15 +16,21 @@ escape: false # Execute inserts execute: false -# Query for insert decrypted data with placeholders (pg: $n) +# Query for insert decrypted data with placeholders (pg: $n, mysql: ?) insert: # Folder from which the keys will be loaded keys_dir: .acrakeys +# Handle MySQL connections +mysql: false + # File for store inserts queries output_file: decrypted.sql +# Handle Postgresql connections +postgresql: false + # Query to fetch data for decryption select: diff --git a/configs/acraproxy.yaml b/configs/acraproxy.yaml index f3e23328b..4215d5982 100644 --- a/configs/acraproxy.yaml +++ b/configs/acraproxy.yaml @@ -37,9 +37,15 @@ disable_user_check: false # dump config dumpconfig: false +# Enable HTTP API +enable_http_api: false + # Folder from which will be loaded keys keys_dir: .acrakeys +# Logging format: plaintext, json or CEF +logging_format: plaintext + # Use raw transport (tcp/unix socket) between acraserver and acraproxy/client (don't use this flag if you not connect to database with ssl/tls no_encryption: false @@ -61,9 +67,6 @@ tls_key: # Expected Server Name (SNI) tls_sni: -# Log to stdout +# Log to stderr v: false -# Turn on zone mode -zonemode: false - diff --git a/configs/acraserver.yaml b/configs/acraserver.yaml index 27b84c335..d0526bd1c 100644 --- a/configs/acraserver.yaml +++ b/configs/acraserver.yaml @@ -1,6 +1,15 @@ +# Path to basic auth passwords. To add user, use: `./acra_genauth --set --user --pwd ` +auth_keys: configs/auth.keys + +# Path to acracensor configuration file +censor_config: + # Expected client id of acraproxy in mode without encryption client_id: +# Time that acraserver will wait (in seconds) on restart before closing all connections +close_connections_timeout: 10 + # Port for AcraServer for http api commands_port: 9090 @@ -22,15 +31,15 @@ db_host: # Port to db db_port: 5432 -# Disable http api -disable_http_api: false - # Turn on http debug server ds: false # dump config dumpconfig: false +# Enable HTTP API +enable_http_api: false + # Escape format for Postgresql bytea data escape_bytea: false @@ -46,6 +55,12 @@ injectedcell: false # Folder from which will be loaded keys keys_dir: .acrakeys +# Logging format: plaintext, json or CEF +logging_format: plaintext + +# Handle MySQL connections +mysql: false + # Use raw transport (tcp/unix socket) between acraserver and acraproxy/client (don't use this flag if you not connect to database with ssl/tls no_encryption: false @@ -58,6 +73,9 @@ poisonshutdown: false # Port for AcraServer port: 9393 +# Handle Postgresql connections (default true) +postgresql: false + # Id that will be sent in secure session server_id: acra_server @@ -76,7 +94,7 @@ tls_key: # Expected Server Name (SNI) tls_sni: -# Log to stdout +# Log to stderr v: false # Acrastruct will stored in whole data cell diff --git a/configs/regenerate.sh b/configs/regenerate.sh old mode 100644 new mode 100755 index c769d8ffb..d8b31d310 --- a/configs/regenerate.sh +++ b/configs/regenerate.sh @@ -2,6 +2,7 @@ go run ./cmd/acraserver/*.go --dumpconfig go run ./cmd/acraproxy/*.go --dumpconfig go run ./cmd/acra_addzone/*.go --dumpconfig +go run ./cmd/acra_configui/*.go --dumpconfig go run ./cmd/acra_rollback/*.go --dumpconfig go run ./cmd/acra_genkeys/*.go --dumpconfig go run ./cmd/acra_genpoisonrecord/*.go --dumpconfig \ No newline at end of file diff --git a/decryptor/base/callbacks.go b/decryptor/base/callbacks.go index 7202a6b2f..2234b48d3 100644 --- a/decryptor/base/callbacks.go +++ b/decryptor/base/callbacks.go @@ -29,6 +29,7 @@ type StopCallback struct{} func (*StopCallback) Call() error { log.Warningln("detected poison record, exit") os.Exit(1) + log.Errorln("executed code after os.Exit") return nil } diff --git a/decryptor/base/decryptor.go b/decryptor/base/decryptor.go index 5ac1ead86..c3f53643a 100644 --- a/decryptor/base/decryptor.go +++ b/decryptor/base/decryptor.go @@ -88,6 +88,7 @@ type Decryptor interface { // get current storage of callbacks for detected poison records GetPoisonCallbackStorage() *PoisonCallbackStorage SetZoneMatcher(*zone.ZoneIdMatcher) + GetZoneMatcher() *zone.ZoneIdMatcher GetMatchedZoneId() []byte MatchZone(byte) bool IsWithZone() bool @@ -95,6 +96,7 @@ type Decryptor interface { IsMatchedZone() bool ResetZoneMatch() IsWholeMatch() bool + SetWholeMatch(bool) DecryptBlock([]byte) ([]byte, error) SkipBeginInBlock(block []byte) ([]byte, error) MatchZoneBlock([]byte) diff --git a/decryptor/mysql/column_field.go b/decryptor/mysql/column_field.go new file mode 100644 index 000000000..9ce3698f9 --- /dev/null +++ b/decryptor/mysql/column_field.go @@ -0,0 +1,169 @@ +package mysql + +import "encoding/binary" + +// ColumnDescription https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41 +type ColumnDescription struct { + changed bool + // field as byte slice + data []byte + Schema []byte + Table []byte + OrgTable []byte + Name []byte + OrgName []byte + Charset uint16 + ColumnLength uint32 + Type uint8 + Flag uint16 + Decimal uint8 + + DefaultValueLength uint64 + DefaultValue []byte +} + +func ParseResultField(data []byte) (*ColumnDescription, error) { + field := &ColumnDescription{} + field.data = data + + var n int + var err error + //skip catalog, always def + pos := 0 + n, err = SkipLengthEncodedString(data) + if err != nil { + return nil, err + } + pos += n + + //schema + field.Schema, _, n, err = LengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + //table + field.Table, _, n, err = LengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + //org_table + field.OrgTable, _, n, err = LengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + //name + field.Name, _, n, err = LengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + //org_name + field.OrgName, _, n, err = LengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + //skip 0x0C constant field + pos += 1 + + //charset + field.Charset = binary.LittleEndian.Uint16(data[pos:]) + pos += 2 + + //column length + field.ColumnLength = binary.LittleEndian.Uint32(data[pos:]) + pos += 4 + + //type + field.Type = data[pos] + pos++ + + //flag + field.Flag = binary.LittleEndian.Uint16(data[pos:]) + pos += 2 + + //decimals 1 + field.Decimal = data[pos] + pos++ + + //filter [0x00][0x00] + pos += 2 + + field.DefaultValue = nil + //if more data, command was field list + if len(data) > pos { + //length of default value lenenc-int + field.DefaultValueLength, _, n, err = LengthEncodedInt(data[pos:]) + if err != nil{ + return nil, err + } + pos += n + + if pos+int(field.DefaultValueLength) > len(data) { + err = ErrMalformPacket + return nil, err + } + + //default value string[$len] + field.DefaultValue = data[pos:(pos + int(field.DefaultValueLength))] + } + return field, nil +} + +func (field *ColumnDescription) IsBinary() bool { + return IsBinaryColumn(field.Type) +} + +// Dump https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41 +func (field *ColumnDescription) Dump() []byte { + if field.data != nil && !field.changed { + return field.data + } + // column description has 7 length encoded strings. each string have 1-4 bytes with their length + // catalog field always has value "def" and 1 byte for length + // one field is constant 0x0C and has 1 byte for length + // left 5 fields may have 8 byte (64bit) for length per field + // (5 * 8) + 4 ("def" + 1 byte for length ) + 1 (0x0C) = 45 + // each of 7 length encoded string fields may have 8 byte (max) for encoded length at start. + // https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41 + l := len(field.Schema) + len(field.Table) + len(field.OrgTable) + len(field.Name) + len(field.OrgName) + len(field.DefaultValue) + 45 + + data := make([]byte, 0, l) + + data = append(data, PutLengthEncodedString([]byte("def"))...) + + data = append(data, PutLengthEncodedString(field.Schema)...) + + data = append(data, PutLengthEncodedString(field.Table)...) + data = append(data, PutLengthEncodedString(field.OrgTable)...) + + data = append(data, PutLengthEncodedString(field.Name)...) + data = append(data, PutLengthEncodedString(field.OrgName)...) + + // length of fixed-length fields + // https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition + data = append(data, 0x0c) + + data = append(data, Uint16ToBytes(field.Charset)...) + data = append(data, Uint32ToBytes(field.ColumnLength)...) + data = append(data, field.Type) + data = append(data, Uint16ToBytes(field.Flag)...) + data = append(data, field.Decimal) + // filler + data = append(data, 0, 0) + + if field.DefaultValue != nil { + data = append(data, Uint64ToBytes(field.DefaultValueLength)...) + data = append(data, field.DefaultValue...) + } + + return data +} diff --git a/decryptor/mysql/decryptor.go b/decryptor/mysql/decryptor.go new file mode 100644 index 000000000..c795e42c6 --- /dev/null +++ b/decryptor/mysql/decryptor.go @@ -0,0 +1,256 @@ +package mysql + +import ( + "bytes" + "io" + "io/ioutil" + + "github.com/cossacklabs/acra/decryptor/base" + "github.com/cossacklabs/acra/decryptor/binary" + "github.com/cossacklabs/acra/decryptor/postgresql" + "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" + "github.com/cossacklabs/acra/utils" + "github.com/cossacklabs/acra/zone" + "github.com/cossacklabs/themis/gothemis/keys" + log "github.com/sirupsen/logrus" +) + +type decryptFunc func([]byte) ([]byte, error) + +type MySQLDecryptor struct { + base.Decryptor + binaryDecryptor *binary.BinaryDecryptor + keyStore keystore.KeyStore + decryptFunc decryptFunc + log *log.Entry +} + +const ( + DECRYPT_WHOLE = "whole_block" + DECRYPT_INLINE = "inline_block" +) + +func NewMySQLDecryptor(pgDecryptor *postgresql.PgDecryptor, keyStore keystore.KeyStore) *MySQLDecryptor { + decryptor := &MySQLDecryptor{keyStore: keyStore, binaryDecryptor: binary.NewBinaryDecryptor(), Decryptor: pgDecryptor} + decryptor.log = log.WithField("decryptor", "mysql") + decryptor.SetWholeMatch(pgDecryptor.IsWholeMatch()) + return decryptor +} + +func (decryptor *MySQLDecryptor) SkipBeginInBlock(block []byte) ([]byte, error) { + n := 0 + for _, c := range block { + if !decryptor.MatchBeginTag(c) { + return []byte{}, base.ErrFakeAcraStruct + } + n++ + if decryptor.IsMatched() { + break + } + } + + if !decryptor.IsMatched() { + return []byte{}, base.ErrFakeAcraStruct + } + return block[n:], nil +} +func (decryptor *MySQLDecryptor) MatchZoneBlock(block []byte) { + for _, c := range block { + if !decryptor.MatchZone(c) { + return + } + } +} +func (decryptor *MySQLDecryptor) BeginTagIndex(block []byte) (int, int) { + if i := utils.FindTag(base.TAG_SYMBOL, decryptor.binaryDecryptor.GetTagBeginLength(), block); i != utils.NOT_FOUND { + return i, decryptor.binaryDecryptor.GetTagBeginLength() + } + return utils.NOT_FOUND, decryptor.GetTagBeginLength() +} + +func (decryptor *MySQLDecryptor) MatchZoneInBlock(block []byte) { + for { + // binary format + i := utils.FindTag(zone.ZONE_TAG_SYMBOL, zone.ZONE_TAG_LENGTH, block) + if i == utils.NOT_FOUND { + break + } else { + if decryptor.keyStore.HasZonePrivateKey(block[i : i+zone.ZONE_ID_BLOCK_LENGTH]) { + decryptor.GetZoneMatcher().SetMatched(block[i : i+zone.ZONE_ID_BLOCK_LENGTH]) + return + } + block = block[i+1:] + } + } + return +} + +func (decryptor *MySQLDecryptor) ReadData(symmetricKey, zoneId []byte, reader io.Reader) ([]byte, error) { + return decryptor.binaryDecryptor.ReadData(symmetricKey, zoneId, reader) +} + +func (decryptor *MySQLDecryptor) ReadSymmetricKey(privateKey *keys.PrivateKey, reader io.Reader) ([]byte, []byte, error) { + symmetricKey, rawData, err := decryptor.binaryDecryptor.ReadSymmetricKey(privateKey, reader) + if err != nil { + return symmetricKey, rawData, err + } + return symmetricKey, rawData, nil +} + +func (decryptor *MySQLDecryptor) getPoisonPrivateKey() (*keys.PrivateKey, error) { + keypair, err := decryptor.keyStore.GetPoisonKeyPair() + if err != nil { + return nil, err + } + return keypair.Private, nil +} + +// CheckPoisonRecord check data from reader on poison records +// added to implement base.Decryptor interface +func (decryptor *MySQLDecryptor) CheckPoisonRecord(reader io.Reader) (bool, error) { + block, err := ioutil.ReadAll(reader) + if err != nil { + return false, err + } + return decryptor.checkPoisonRecord(block) +} + +func (decryptor *MySQLDecryptor) checkPoisonRecord(block []byte) (bool, error) { + decryptor.Reset() + data, err := decryptor.SkipBeginInBlock(block) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantSkipBeginInBlock). + Debugln("Can't skip begin tag in block") + return false, nil + } + log.Debugln("Check block on poison") + _, err = decryptor.decryptBlock(bytes.NewReader(data), nil, decryptor.getPoisonPrivateKey) + if err == nil { + log.Warningln("Recognized poison record") + if decryptor.GetPoisonCallbackStorage().HasCallbacks() { + log.Debugln("Check poison records") + if err := decryptor.GetPoisonCallbackStorage().Call(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantHandleRecognizedPoisonRecord). + Errorln("Unexpected error in poison record callbacks") + } + log.Debugln("Processed all callbacks on poison record") + } + return true, err + } + return false, nil +} + +// poisonCheck find acrastructs in block and try to detect poison record +func (decryptor *MySQLDecryptor) poisonCheck(block []byte) error { + index := 0 + for { + beginTagIndex, _ := decryptor.BeginTagIndex(block[index:]) + if beginTagIndex == utils.NOT_FOUND { + break + } else { + log.Debugln("Found acrastruct") + poisoned, err := decryptor.checkPoisonRecord(block[index+beginTagIndex:]) + if poisoned { + return base.ErrPoisonRecord + } + return err + } + index++ + } + return nil +} + +type getKeyFunc func() (*keys.PrivateKey, error) + +// decryptBlock try to process data after BEGIN_TAG, decrypt and return result +func (decryptor *MySQLDecryptor) decryptBlock(reader io.Reader, id []byte, keyFunc getKeyFunc) ([]byte, error) { + privateKey, err := keyFunc() + if err != nil { + decryptor.log.Warningln("Can't read private key") + return []byte{}, err + } + key, _, err := decryptor.ReadSymmetricKey(privateKey, reader) + if err != nil { + decryptor.log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptSymmetricKey).Warningln("Can't unwrap symmetric key") + return []byte{}, err + } + data, err := decryptor.ReadData(key, id, reader) + if err != nil { + decryptor.log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary).Warningln("Can't decrypt data with unwrapped symmetric key") + return []byte{}, err + } + return data, nil +} + +func (decryptor *MySQLDecryptor) SetWholeMatch(value bool) { + if value { + decryptor.decryptFunc = decryptor.decryptWholeBlock + decryptor.log = decryptor.log.WithField("decrypt_mode", DECRYPT_WHOLE) + } else { + decryptor.decryptFunc = decryptor.decryptInlineBlock + decryptor.log = decryptor.log.WithField("decrypt_mode", DECRYPT_INLINE) + } +} + +func (decryptor *MySQLDecryptor) decryptWholeBlock(block []byte) ([]byte, error) { + var err error + if err := decryptor.poisonCheck(block); err != nil { + return nil, err + } + decryptor.Reset() + if !decryptor.IsWithZone() || decryptor.IsMatchedZone() { + block, err = decryptor.SkipBeginInBlock(block) + if err != nil { + return nil, err + } + newData, err := decryptor.decryptBlock(bytes.NewReader(block), decryptor.GetMatchedZoneId(), decryptor.GetPrivateKey) + if decryptor.IsWithZone() && err == nil && len(newData) != len(block) { + decryptor.ResetZoneMatch() + } + return newData, err + } else { + decryptor.MatchZoneBlock(block) + return block, nil + } +} + +func (decryptor *MySQLDecryptor) decryptInlineBlock(block []byte) ([]byte, error) { + if err := decryptor.poisonCheck(block); err != nil { + return nil, err + } + var output bytes.Buffer + index := 0 + log.Debugf("block len %v", len(block)) + if decryptor.IsWithZone() && !decryptor.IsMatchedZone() { + decryptor.MatchZoneInBlock(block) + return block, nil + } + for index < len(block) { + log.Debugf("index=%v", index) + decryptor.log.Debugf("index: %v", index) + beginTagIndex, tagLength := decryptor.BeginTagIndex(block[index:]) + if beginTagIndex == utils.NOT_FOUND { + output.Write(block[index:]) + return output.Bytes(), nil + } + output.Write(block[index : index+beginTagIndex]) + index += beginTagIndex + blockReader := bytes.NewReader(block[index+tagLength:]) + decrypted, err := decryptor.decryptBlock(blockReader, decryptor.GetMatchedZoneId(), decryptor.GetPrivateKey) + if err != nil { + output.Write(block[index : index+1]) + index++ + decryptor.log.Debugln("can't decrypt block") + continue + } + index += tagLength + (len(block[beginTagIndex+tagLength:]) - blockReader.Len()) + output.Write(decrypted) + decryptor.ResetZoneMatch() + } + return output.Bytes(), nil +} + +func (decryptor *MySQLDecryptor) DecryptBlock(block []byte) ([]byte, error) { + return decryptor.decryptFunc(block) +} diff --git a/decryptor/mysql/decryptor_test.go b/decryptor/mysql/decryptor_test.go new file mode 100644 index 000000000..0a868e62a --- /dev/null +++ b/decryptor/mysql/decryptor_test.go @@ -0,0 +1,112 @@ +package mysql + +import ( + "testing" + + "crypto/rand" + + "github.com/cossacklabs/acra/decryptor/base" + "github.com/cossacklabs/acra/decryptor/binary" + "github.com/cossacklabs/acra/decryptor/postgresql" + "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/poison" + "github.com/cossacklabs/themis/gothemis/keys" +) + +type testKeystore struct { + PoisonKeypair *keys.Keypair +} + +func (keystore *testKeystore) GetPoisonKeyPair() (*keys.Keypair, error) { + return keystore.PoisonKeypair, nil +} + +func (keystore *testKeystore) GetPrivateKey(id []byte) (*keys.PrivateKey, error) { + return nil, nil +} +func (keystore *testKeystore) GetPeerPublicKey(id []byte) (*keys.PublicKey, error) { + return nil, nil +} +func (keystore *testKeystore) GetZonePrivateKey(id []byte) (*keys.PrivateKey, error) { + return nil, nil +} +func (keystore *testKeystore) HasZonePrivateKey(id []byte) bool { + return true +} +func (keystore *testKeystore) GetServerDecryptionPrivateKey(id []byte) (*keys.PrivateKey, error) { + return nil, nil +} +func (keystore *testKeystore) GenerateZoneKey() ([]byte, []byte, error) { + return nil, nil, nil +} +func (keystore *testKeystore) GenerateProxyKeys(id []byte) error { + return nil +} +func (keystore *testKeystore) GenerateServerKeys(id []byte) error { + return nil +} +func (keystore *testKeystore) GenerateDataEncryptionKeys(id []byte) error { + return nil +} +func (keystore *testKeystore) GetAuthKey(remove bool) ([]byte, error) { + return nil, nil +} +func (keystore *testKeystore) Reset() {} + +func getDecryptor(keystore keystore.KeyStore) *MySQLDecryptor { + dataDecryptor := binary.NewBinaryDecryptor() + clientId := []byte("some id") + pgDecryptor := postgresql.NewPgDecryptor(clientId, dataDecryptor) + decryptor := NewMySQLDecryptor(pgDecryptor, keystore) + + poisonCallbackStorage := base.NewPoisonCallbackStorage() + decryptor.SetPoisonCallbackStorage(poisonCallbackStorage) + return decryptor +} + +func TestMySQLDecryptor_CheckPoisonRecord_Inline(t *testing.T) { + poisonKeypair, err := keys.New(keys.KEYTYPE_EC) + if err != nil { + panic(err) + } + keystore := &testKeystore{PoisonKeypair: poisonKeypair} + decryptor := getDecryptor(keystore) + + part1 := make([]byte, 1024) + part2 := make([]byte, 1024) + if _, err := rand.Read(part1); err != nil { + t.Fatal(err) + } + if _, err := rand.Read(part2); err != nil { + t.Fatal(err) + } + + poisonRecord, err := poison.CreatePoisonRecord(keystore, 1024) + if err != nil { + t.Fatal(err) + } + + testData := append(part1, append(poisonRecord, part2...)...) + err = decryptor.poisonCheck(testData) + if err != base.ErrPoisonRecord { + t.Fatal("expected ErrPoisonRecord") + } +} + +func TestMySQLDecryptor_CheckPoisonRecord_Block(t *testing.T) { + poisonKeypair, err := keys.New(keys.KEYTYPE_EC) + if err != nil { + panic(err) + } + keystore := &testKeystore{PoisonKeypair: poisonKeypair} + decryptor := getDecryptor(keystore) + + poisonRecord, err := poison.CreatePoisonRecord(keystore, 1024) + if err != nil { + t.Fatal(err) + } + err = decryptor.poisonCheck(poisonRecord) + if err != base.ErrPoisonRecord { + t.Fatal("expected ErrPoisonRecord") + } +} diff --git a/decryptor/mysql/error.go b/decryptor/mysql/error.go new file mode 100644 index 000000000..110b285ee --- /dev/null +++ b/decryptor/mysql/error.go @@ -0,0 +1,46 @@ +package mysql + +type SqlError struct { + Code uint16 + Message string + State string +} + +const ( + // https://dev.mysql.com/doc/refman/5.5/en/error-messages-server.html#error_er_query_interrupted + ER_QUERY_INTERRUPTED_CODE = 1317 + ER_QUERY_INTERRUPTED_STATE = "70100" +) + +func newQueryInterruptedError() *SqlError { + e := new(SqlError) + e.Code = ER_QUERY_INTERRUPTED_CODE + e.State = ER_QUERY_INTERRUPTED_STATE + e.Message = "Query execution was interrupted" + return e +} + +// NewQueryInterruptedError return packed QueryInterrupted error +// https://dev.mysql.com/doc/internals/en/packet-ERR_Packet.html +func NewQueryInterruptedError(isProtocol41 bool) []byte { + mysqlError := newQueryInterruptedError() + var data []byte + if isProtocol41 { + // 1 byte ERR_PACKET flag + 2 bytes of error code = 3 + data = make([]byte, 0, 3+len(mysqlError.Message)) + } else { + // 1 byte ERR_PACKET flag + 2 bytes of error code + 6 bytes of state (protocol41) = 9 + data = make([]byte, 0, 9+len(mysqlError.Message)) + } + + data = append(data, ERR_PACKET) + data = append(data, byte(mysqlError.Code), byte(mysqlError.Code>>8)) + + if isProtocol41 { + data = append(data, '#') + data = append(data, mysqlError.State...) + } + + data = append(data, mysqlError.Message...) + return data +} diff --git a/decryptor/mysql/packet.go b/decryptor/mysql/packet.go new file mode 100644 index 000000000..8c5ab25c9 --- /dev/null +++ b/decryptor/mysql/packet.go @@ -0,0 +1,205 @@ +package mysql + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net" + "errors" +) + +const ( + // CLIENT_PROTOCOL_41 - https://dev.mysql.com/doc/internals/en/capability-flags.html#flag-CLIENT_PROTOCOL_41 + CLIENT_PROTOCOL_41 = 0x00000200 + // SSL_REQUEST - https://dev.mysql.com/doc/internals/en/capability-flags.html#flag-CLIENT_SSL + SSL_REQUEST = 0x00000800 + // https://dev.mysql.com/doc/internals/en/capability-flags.html#flag-CLIENT_DEPRECATE_EOF - 0x1000000 + CLIENT_DEPRECATE_EOF = 1 << 6 +) + +const ( + // OK_PACKET - https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html + OK_PACKET = 0x00 + // EOF_PACKET - https://dev.mysql.com/doc/internals/en/packet-EOF_Packet.html + EOF_PACKET = 0xfe + ERR_PACKET = 0xff +) + +const ( + // PACKET_HEADER_SIZE https://dev.mysql.com/doc/internals/en/mysql-packet.html#idm140406396409840 + PACKET_HEADER_SIZE = 4 + // SEQUENCE_ID_INDEX last byte of header https://dev.mysql.com/doc/internals/en/mysql-packet.html#idm140406396409840 + SEQUENCE_ID_INDEX = 3 +) + +var ErrPacketHasNotExtendedCapabilities = errors.New("packet hasn't extended capabilities") + +type Dumper interface { + Dump() []byte +} + +type ByteArrayDump []byte + +func (array ByteArrayDump) Dump() []byte { + return array +} + +// MysqlPacket struct that store header and payload, read it from connectino +type MysqlPacket struct { + header []byte + data []byte +} + +// NewMysqlPacket +func NewMysqlPacket() *MysqlPacket { + // https://dev.mysql.com/doc/internals/en/mysql-packet.html#idm140406396409840 + // 3 bytes payload length and 1 byte of sequence_id + return &MysqlPacket{header: make([]byte, PACKET_HEADER_SIZE)} +} + +// GetPacketPayloadLength +func (packet *MysqlPacket) GetPacketPayloadLength() int { + // first 3 bytes of header + // https://dev.mysql.com/doc/internals/en/mysql-packet.html#idm140406396409840 + return int(uint32(packet.header[0]) | uint32(packet.header[1])<<8 | uint32(packet.header[2])<<16) +} + +// GetSequenceNumber return as byte +func (packet *MysqlPacket) GetSequenceNumber() byte { + return packet.header[SEQUENCE_ID_INDEX] +} + +// GetData return packet payload +func (packet *MysqlPacket) GetData() []byte { + return packet.data +} + +// SetData replace packet data with newData and update payload length in header +func (packet *MysqlPacket) SetData(newData []byte) { + packet.data = newData + newSize := len(newData) + // update payload size, first 3 bytes of header + // https://dev.mysql.com/doc/internals/en/mysql-packet.html#idm140406396409840 + packet.header[0] = byte(newSize) + packet.header[1] = byte(newSize >> 8) + packet.header[2] = byte(newSize >> 16) +} + +// readPacket read header to struct and return payload as return result or error +func (packet *MysqlPacket) readPacket(connection net.Conn) ([]byte, error) { + if _, err := connection.Read(packet.header); err != nil { + return nil, err + } + + length := packet.GetPacketPayloadLength() + if length < 1 { + return nil, fmt.Errorf("invalid payload length %d", length) + } + + data := make([]byte, length) + if _, err := io.ReadFull(connection, data); err != nil { + return nil, err + } else { + if length < MaxPayloadLen { + return data, nil + } + + var buf []byte + buf, err = packet.readPacket(connection) + if err != nil { + return nil, err + } else { + return append(data, buf...), nil + } + } +} +func (packet *MysqlPacket) Dump() []byte { + return append(packet.header, packet.data...) +} + +// ReadPacket header and payload from connection or return error +func (packet *MysqlPacket) ReadPacket(connection net.Conn) error { + data, err := packet.readPacket(connection) + if err == nil { + packet.data = data + } + return err +} + +// IsEOF return true if packet is OK_PACKET or EOF_PACHET +func (packet *MysqlPacket) IsEOF() bool { + // https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html + // https://dev.mysql.com/doc/internals/en/packet-EOF_Packet.html + isOkPacket := packet.data[0] == OK_PACKET && packet.GetPacketPayloadLength() > 7 + isEOFPacket := packet.data[0] == EOF_PACKET && packet.GetPacketPayloadLength() < 9 + return isOkPacket || isEOFPacket +} + +// IsErr return true if packet has ERR_PACKET flag +func (packet *MysqlPacket) IsErr() bool { + return packet.data[0] == ERR_PACKET +} + +func (packet *MysqlPacket) getServerCapabilities() int { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#idm140437490034448 + endOfServerVersion := bytes.Index(packet.data[1:], []byte{0}) + 2 // 1 first byte of protocol version and 1 to point to next byte + // 4 bytes connection string + 8 bytes of auth plugin + 1 byte filler + rawCapabilities := packet.data[endOfServerVersion+13 : endOfServerVersion+13+2] + return int(binary.LittleEndian.Uint16(rawCapabilities)) +} + +func (packet *MysqlPacket) getServerCapabilitiesExtended() (int, error) { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#idm140437490034448 + endOfServerVersion := bytes.Index(packet.data[1:], []byte{0}) + 2 // 1 first byte of protocol version and 1 to point to next byte + // 4 bytes connection string + 8 bytes of auth plugin + 1 byte filler + baseCapabilitiesOffset := endOfServerVersion + 13 + // 2 bytes of base capabilities + 1 byte character set + 2 bytes of status flags + capabilitiesOffset := baseCapabilitiesOffset + 2 + 3 + if len(packet.data) < capabilitiesOffset+2 { + return 0, ErrPacketHasNotExtendedCapabilities + } + rawCapabilities := packet.data[capabilitiesOffset : capabilitiesOffset+2] + return int(binary.LittleEndian.Uint16(rawCapabilities)), nil +} + +func (packet *MysqlPacket) ServerSupportProtocol41() bool { + capabilities := packet.getServerCapabilities() + return (capabilities & CLIENT_PROTOCOL_41) > 0 +} + +func (packet *MysqlPacket) getClientCapabilities() int { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#idm140437489940880 + return int(binary.LittleEndian.Uint16(packet.data[:2])) +} + +func (packet *MysqlPacket) ClientSupportProtocol41() bool { + capabilities := packet.getClientCapabilities() + return (capabilities & CLIENT_PROTOCOL_41) > 0 +} + +// IsSSLRequest return true if SSL_REQUEST flag up +func (packet *MysqlPacket) IsSSLRequest() bool { + capabilities := packet.getClientCapabilities() + return (capabilities & SSL_REQUEST) > 0 +} + +// IsClientDeprecatedEOF return true if flag set +// https://dev.mysql.com/doc/internals/en/capability-flags.html#flag-CLIENT_DEPRECATE_EOF +func (packet *MysqlPacket) IsClientDeprecateEOF() bool { + capabilities, err := packet.getServerCapabilitiesExtended() + if err != nil { + return false + } + return (capabilities & CLIENT_DEPRECATE_EOF) > 0 +} + +// ReadPacket from connection and return MysqlPacket struct with data or error +func ReadPacket(connection net.Conn) (*MysqlPacket, error) { + packet := NewMysqlPacket() + err := packet.ReadPacket(connection) + if err != nil { + return nil, err + } + return packet, nil +} diff --git a/decryptor/mysql/response_proxy.go b/decryptor/mysql/response_proxy.go new file mode 100644 index 000000000..9000cd51b --- /dev/null +++ b/decryptor/mysql/response_proxy.go @@ -0,0 +1,547 @@ +package mysql + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/cossacklabs/acra/decryptor/base" + "github.com/cossacklabs/acra/acracensor" + "github.com/cossacklabs/acra/logging" + log "github.com/sirupsen/logrus" +) + +const ( + // MaxPayloadLen https://dev.mysql.com/doc/internals/en/mysql-packet.html + // each packet splits into packets of this size + MaxPayloadLen int = 1<<24 - 1 + CLIENT_WAIT_DB_TLS_HANDSHAKE = 5 +) + +const ( + COM_SLEEP byte = iota + COM_QUIT + COM_INIT_DB + COM_QUERY + COM_FIELD_LIST + COM_CREATE_DB + COM_DROP_DB + COM_REFRESH + COM_SHUTDOWN + COM_STATISTICS + COM_PROCESS_INFO + COM_CONNECT + COM_PROCESS_KILL + COM_DEBUG + COM_PING + COM_TIME + COM_DELAYED_INSERT + COM_CHANGE_USER + COM_BINLOG_DUMP + COM_TABLE_DUMP + COM_CONNECT_OUT + COM_REGISTER_SLAVE + COM_STMT_PREPARE + COM_STMT_EXECUTE + COM_STMT_SEND_LONG_DATA + COM_STMT_CLOSE + COM_STMT_RESET + COM_SET_OPTION + COM_STMT_FETCH + COM_DAEMON + COM_BINLOG_DUMP_GTID + COM_RESET_CONNECTION +) + +// Binary ColumnTypes https://dev.mysql.com/doc/internals/en/com-query-response.html#column-type +const ( + MYSQL_TYPE_DECIMAL byte = iota + MYSQL_TYPE_TINY + MYSQL_TYPE_SHORT + MYSQL_TYPE_LONG + MYSQL_TYPE_FLOAT + MYSQL_TYPE_DOUBLE + MYSQL_TYPE_NULL + MYSQL_TYPE_TIMESTAMP + MYSQL_TYPE_LONGLONG + MYSQL_TYPE_INT24 + MYSQL_TYPE_DATE + MYSQL_TYPE_TIME + MYSQL_TYPE_DATETIME + MYSQL_TYPE_YEAR + MYSQL_TYPE_NEWDATE + MYSQL_TYPE_VARCHAR + MYSQL_TYPE_BIT +) + +const ( + MYSQL_TYPE_NEWDECIMAL byte = iota + 0xf6 + MYSQL_TYPE_ENUM + MYSQL_TYPE_SET + MYSQL_TYPE_TINY_BLOB + MYSQL_TYPE_MEDIUM_BLOB + MYSQL_TYPE_LONG_BLOB + MYSQL_TYPE_BLOB + MYSQL_TYPE_VAR_STRING + MYSQL_TYPE_STRING + MYSQL_TYPE_GEOMETRY +) + +const ( + NOT_NULL_FLAG = 1 + PRI_KEY_FLAG = 2 + UNIQUE_KEY_FLAG = 4 + BLOB_FLAG = 16 + UNSIGNED_FLAG = 32 + ZEROFILL_FLAG = 64 + BINARY_FLAG = 128 + ENUM_FLAG = 256 + AUTO_INCREMENT_FLAG = 512 + TIMESTAMP_FLAG = 1024 + SET_FLAG = 2048 + NUM_FLAG = 32768 + PART_KEY_FLAG = 16384 + GROUP_FLAG = 32768 + UNIQUE_FLAG = 65536 +) +const ( + TLS_NONE = iota + TLS_CLIENT_SWITCH + TLS_DB_COMPLETE +) + +func IsBinaryColumn(value byte) bool { + isBlob := value > MYSQL_TYPE_TINY_BLOB && value < MYSQL_TYPE_BLOB + isString := value == MYSQL_TYPE_VAR_STRING || value == MYSQL_TYPE_STRING + return isString || isBlob || value == MYSQL_TYPE_VARCHAR +} + +type ResponseHandler func(packet *MysqlPacket, dbConnection, clientConnection net.Conn) error + +func defaultResponseHandler(packet *MysqlPacket, dbConnection, clientConnection net.Conn) error { + if _, err := clientConnection.Write(packet.Dump()); err != nil { + return err + } + return nil +} + +type MysqlHandler struct { + responseHandler ResponseHandler + clientSequenceNumber int + serverSequenceNumber int + clientProtocol41 bool + serverProtocol41 bool + // clientDeprecateEOF if false then expect EOF on response result as terminator otherwise not + clientDeprecateEOF bool + decryptor base.Decryptor + acracensor acracensor.AcracensorInterface + isTLSHandshake bool + dbTLSHandshakeFinished chan bool + clientConnection net.Conn + dbConnection net.Conn + tlsConfig *tls.Config +} + +func NewMysqlHandler(decryptor base.Decryptor, dbConnection, clientConnection net.Conn, tlsConfig *tls.Config, censor acracensor.AcracensorInterface) (*MysqlHandler, error) { + return &MysqlHandler{isTLSHandshake: false, dbTLSHandshakeFinished: make(chan bool), clientDeprecateEOF:false, decryptor: decryptor, responseHandler: defaultResponseHandler, acracensor: censor, clientConnection: clientConnection, dbConnection: dbConnection, tlsConfig: tlsConfig}, nil +} + +func (handler *MysqlHandler) setQueryHandler(callback ResponseHandler) { + handler.responseHandler = callback +} +func (handler *MysqlHandler) resetQueryHandler() { + handler.responseHandler = defaultResponseHandler +} + +func (handler *MysqlHandler) getResponseHandler() ResponseHandler { + return handler.responseHandler +} + +func (handler *MysqlHandler) ClientToDbProxy(errCh chan<- error) { + clientLog := log.WithField("proxy", "client") + clientLog.Debugln("Start proxy client's requests") + firstPacket := true + for { + packet, err := ReadPacket(handler.clientConnection) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantReadFromClient). + Debugln("Can't read packet from client") + errCh <- err + return + } + if firstPacket { + firstPacket = false + handler.clientProtocol41 = packet.ClientSupportProtocol41() + if packet.IsSSLRequest() { + tlsConnection := tls.Server(handler.clientConnection, handler.tlsConfig) + if err := tlsConnection.Handshake(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantInitializeTLS). + Errorln("error in tls handshake with client") + errCh <- err + return + } + log.Debugln("switched to tls with client. wait switching with db") + handler.isTLSHandshake = true + handler.clientConnection = tlsConnection + if _, err := handler.dbConnection.Write(packet.Dump()); err != nil { + clientLog.Debugln("can't write send packet to db") + errCh <- err + return + } + // stop reading and init switching to tls + handler.dbConnection.SetReadDeadline(time.Now()) + // we should wait when db proxy part will finish handshake to avoid case when new packets from client + // will be proxied in this function to db before handshake will be completed + select { + case <-handler.dbTLSHandshakeFinished: + log.Debugln("switch to tls complete on client proxy side") + continue + case <-time.NewTicker(time.Second * CLIENT_WAIT_DB_TLS_HANDSHAKE).C: + clientLog.Errorln("timeout on tls handshake with db") + errCh <- errors.New("handshake timeout") + return + } + continue + } + } + handler.clientSequenceNumber = int(packet.GetSequenceNumber()) + clientLog = clientLog.WithField("sequence_number", handler.clientSequenceNumber) + clientLog.Debugln("New packet") + inOutput := packet.Dump() + data := packet.GetData() + cmd := data[0] + data = data[1:] + + switch cmd { + case COM_QUIT: + clientLog.Debugln("Close connections on COM_QUIT command") + handler.clientConnection.Close() + handler.dbConnection.Close() + errCh <- io.EOF + return + case COM_QUERY: + sqlQuery := string(data) + if err := handler.acracensor.HandleQuery(sqlQuery); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCensorQueryIsNotAllowed). + Errorln("Error on acracensor check") + errPacket := NewQueryInterruptedError(handler.clientProtocol41) + packet.SetData(errPacket) + if _, err := handler.clientConnection.Write(packet.Dump()); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantWriteToClient). + Errorln("Can't write response with error to client") + } + continue + } + clientLog.WithField("sql", sqlQuery).Debugln("com_query") + handler.setQueryHandler(handler.QueryResponseHandler) + break + case COM_STMT_PREPARE, COM_STMT_EXECUTE, COM_STMT_CLOSE, COM_STMT_SEND_LONG_DATA, COM_STMT_RESET: + fallthrough + default: + clientLog.Debugf("Command %d not supported now", cmd) + } + if _, err := handler.dbConnection.Write(inOutput); err != nil { + clientLog.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantWriteToDB). + Debugln("Can't write send packet to db") + errCh <- err + return + } + } +} + +// TODO: remove because it's not needed +func (handler *MysqlHandler) isFieldToDecrypt(field *ColumnDescription) bool { + switch field.Type { + case MYSQL_TYPE_VARCHAR, MYSQL_TYPE_TINY_BLOB, MYSQL_TYPE_MEDIUM_BLOB, MYSQL_TYPE_LONG_BLOB, MYSQL_TYPE_BLOB, + MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING: + return true + default: + return false + } +} + +func (handler *MysqlHandler) processTextDataRow(rowData []byte, fields []*ColumnDescription) ([]byte, error) { + var err error + var value []byte + var pos int = 0 + var n int = 0 + var output []byte + var fieldLogger *log.Entry + log.Debugln("Process fields in text data row") + for i := range fields { + fieldLogger = log.WithField("field_index", i) + value, _, n, err = LengthEncodedString(rowData[pos:]) + if err != nil { + return nil, err + } + if handler.isFieldToDecrypt(fields[i]) { + decryptedValue, err := handler.decryptor.DecryptBlock(value) + if err != nil { + fieldLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). + Errorln("Can't decrypt binary data") + } + if err == nil && len(decryptedValue) != len(value) { + fieldLogger.Debugln("Update with decrypted value") + output = append(output, PutLengthEncodedString(decryptedValue)...) + } else { + fieldLogger.Debugln("Leave value as is") + output = append(output, rowData[pos:pos+n]...) + } + pos += n + continue + } + fieldLogger.Debugln("Field is not binary") + + output = append(output, rowData[pos:pos+n]...) + pos += n + } + log.Debugln("Finish processing text data row") + + return output, nil +} + +func (handler *MysqlHandler) processBinaryDataRow(rowData []byte, fields []*ColumnDescription) ([]byte, error) { + pos := 0 + var n int + var err error + var value []byte + var output []byte + for i := range fields { + if handler.isFieldToDecrypt(fields[i]) { + value, _, n, err = LengthEncodedString(rowData[pos:]) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). + Errorln("Can't handle length encoded string binary value") + return nil, err + } + decryptedValue, err := handler.decryptor.DecryptBlock(value) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). + Errorln("Can't decrypt binary data") + return nil, err + } + if len(value) != len(decryptedValue) { + output = append(output, PutLengthEncodedString(decryptedValue)...) + } else { + output = append(output, rowData[pos:pos+n]...) + } + + pos += n + continue + } + // https://dev.mysql.com/doc/internals/en/binary-protocol-value.html + switch fields[i].Type { + case MYSQL_TYPE_NULL: + continue + + case MYSQL_TYPE_TINY: + output = append(output, rowData[pos]) + pos++ + continue + + case MYSQL_TYPE_SHORT, MYSQL_TYPE_YEAR: + output = append(output, rowData[pos:pos+2]...) + pos += 2 + continue + + case MYSQL_TYPE_INT24, MYSQL_TYPE_LONG: + output = append(output, rowData[pos:pos+4]...) + pos += 4 + continue + + case MYSQL_TYPE_LONGLONG: + output = append(output, rowData[pos:pos+8]...) + pos += 8 + continue + + case MYSQL_TYPE_FLOAT: + output = append(output, rowData[pos:pos+4]...) + pos += 4 + continue + + case MYSQL_TYPE_DOUBLE: + output = append(output, rowData[pos:pos+8]...) + pos += 8 + continue + + case MYSQL_TYPE_DECIMAL, MYSQL_TYPE_NEWDECIMAL, + MYSQL_TYPE_BIT, MYSQL_TYPE_ENUM, MYSQL_TYPE_SET, MYSQL_TYPE_GEOMETRY: + value, _, n, err = LengthEncodedString(rowData[pos:]) + output = append(output, rowData[pos:pos+n]...) + pos += n + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantDecryptBinary). + Errorln("Can't handle length encoded string non binary value") + return nil, err + } + continue + case MYSQL_TYPE_DATE, MYSQL_TYPE_NEWDATE, MYSQL_TYPE_TIMESTAMP, MYSQL_TYPE_DATETIME, MYSQL_TYPE_TIME: + _, _, n, err = LengthEncodedInt(rowData[pos:]) + if err != nil { + return nil, err + } + output = append(output, rowData[pos:pos+n]...) + pos += n + continue + default: + return nil, fmt.Errorf("while decrypting MySQL query found unknown FieldType %d %s", fields[i].Type, fields[i].Name) + } + } + return output, nil +} + +func (handler *MysqlHandler) expectEOFOnColumnDefinition()bool { + return !handler.clientDeprecateEOF +} + +func (handler *MysqlHandler) QueryResponseHandler(packet *MysqlPacket, dbConnection, clientConnection net.Conn) (err error) { + log.Debugln("Query handler") + handler.resetQueryHandler() + handler.decryptor.Reset() + handler.decryptor.ResetZoneMatch() + // read fields + var fields []*ColumnDescription + var binaryFieldIndexes []int + // first byte of payload is field count + // https://dev.mysql.com/doc/internals/en/com-query-response.html#text-resultset + fieldCount := int(packet.GetData()[0]) + if fieldCount == OK_PACKET || fieldCount == ERR_PACKET { + log.Debugln("Error or empty response packet") + if _, err := clientConnection.Write(packet.Dump()); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantWriteToClient). + Errorln("Can't proxy output") + return err + } + return nil + } + output := []Dumper{packet} + log.Debugln("Read column descriptions") + for i := 0; ; i++ { + log.WithField("column_index", i).Debugln("read column description") + fieldPacket, err := ReadPacket(dbConnection) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantProcessColumn). + Errorln("Can't read packet with column description") + return err + } + output = append(output, fieldPacket) + if handler.expectEOFOnColumnDefinition(){ + if fieldPacket.IsEOF() { + if i != fieldCount { + return ErrMalformPacket + } + break + } + } else { + if i == fieldCount { + break + } + } + log.WithField("column_index", i).Debugln("parse field") + field, err := ParseResultField(fieldPacket.GetData()) + if err != nil { + return err + } + if field.IsBinary() { + binaryFieldIndexes = append(binaryFieldIndexes, i) + } + fields = append(fields, field) + } + + log.Debugln("Read data rows") + var dataLog *log.Entry + // read data packets + for i := 0; ; i++ { + dataLog = log.WithField("data_row_index", i) + dataLog.Debugln("read data row") + fieldDataPacket, err := ReadPacket(dbConnection) + if err != nil { + return err + } + output = append(output, fieldDataPacket) + if fieldDataPacket.IsEOF() { + break + } + + dataLength := fieldDataPacket.GetPacketPayloadLength() + dataLog.Debugln("Process data row") + + newData, err := handler.processTextDataRow(fieldDataPacket.GetData(), fields) + if err != nil { + dataLog.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantProcessRow). + Debugln("Can't process text data row") + return err + } + // decrypted data always less than ecrypted + if len(newData) < dataLength { + dataLog.WithFields(log.Fields{"oldLength": dataLength, "newLength": len(newData)}).Debugln("update row data") + fieldDataPacket.SetData(newData) + } + } + + // proxy output + log.Debugln("proxy output") + for _, dumper := range output { + if _, err := clientConnection.Write(dumper.Dump()); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantWriteToClient). + Errorln("can't proxy output") + return err + } + } + log.Debugln("query handler finish") + return nil +} + +func (handler *MysqlHandler) DbToClientProxy(errCh chan<- error) { + serverLog := log.WithField("proxy", "server") + serverLog.Debugln("Start proxy db responses") + firstPacket := true + var responseHandler ResponseHandler + for { + packet, err := ReadPacket(handler.dbConnection) + if err != nil { + if netErr, ok := err.(net.Error); ok { + if netErr.Timeout() && handler.isTLSHandshake { + // reset deadline + handler.dbConnection.SetReadDeadline(time.Time{}) + tlsConnection := tls.Client(handler.dbConnection, handler.tlsConfig) + if err := tlsConnection.Handshake(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantInitializeTLS). + Errorln("error in tls handshake with db") + errCh <- err + return + } + log.Debugln("switched to tls with db") + handler.dbConnection = tlsConnection + handler.dbTLSHandshakeFinished <- true + continue + } + } + log.Debugln("can't read packet from server") + errCh <- err + return + } + log.WithField("sequence_number", packet.GetSequenceNumber()).Debugln("new packet from db to client") + if packet.IsErr() { + handler.resetQueryHandler() + } + if firstPacket { + firstPacket = false + handler.serverProtocol41 = packet.ServerSupportProtocol41() + handler.clientDeprecateEOF = packet.IsClientDeprecateEOF() + serverLog.Debugf("set support protocol 41 %v", handler.serverProtocol41) + } + responseHandler = handler.getResponseHandler() + err = responseHandler(packet, handler.dbConnection, handler.clientConnection) + if err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorResponseProxyCantWriteToServer). + Errorln("Error in responseHandler") + errCh <- err + return + } + + } +} diff --git a/decryptor/mysql/utils.go b/decryptor/mysql/utils.go new file mode 100644 index 000000000..dd2278bda --- /dev/null +++ b/decryptor/mysql/utils.go @@ -0,0 +1,147 @@ +package mysql + +import ( + "errors" + "io" +) + +var ErrMalformPacket = errors.New("Malform packet error") + +// LengthEncodedInt https://dev.mysql.com/doc/internals/en/integer.html#packet-Protocol::LengthEncodedInteger +func LengthEncodedInt(data []byte) (num uint64, isNull bool, n int, err error) { + if len(data) == 0 { + return uint64(0), false, 0, ErrMalformPacket + } + switch data[0] { + + // 251: NULL + case 0xfb: + n = 1 + isNull = true + return + + // 252: value of following 2 + case 0xfc: + if len(data) < 3 { + return uint64(0), false, 0, ErrMalformPacket + } + num = uint64(data[1]) | uint64(data[2])<<8 + n = 3 + return + + // 253: value of following 3 + case 0xfd: + if len(data) < 4{ + return uint64(0), false, 0, ErrMalformPacket + } + num = uint64(data[1]) | uint64(data[2])<<8 | uint64(data[3])<<16 + n = 4 + return + + // 254: value of following 8 + case 0xfe: + if len(data) < 9 { + return uint64(0), false, 0, ErrMalformPacket + } + num = uint64(data[1]) | uint64(data[2])<<8 | uint64(data[3])<<16 | + uint64(data[4])<<24 | uint64(data[5])<<32 | uint64(data[6])<<40 | + uint64(data[7])<<48 | uint64(data[8])<<56 + n = 9 + return + } + + // 0-250: value of first byte + num = uint64(data[0]) + n = 1 + return +} + +// LengthEncodedString https://dev.mysql.com/doc/internals/en/string.html#packet-Protocol::LengthEncodedString +func LengthEncodedString(data []byte) ([]byte, bool, int, error) { + // Get length + num, isNull, n, err := LengthEncodedInt(data) + if num < 1 { + return nil, isNull, n, err + } + + n += int(num) + + // Check data length + if len(data) >= n { + return data[n-int(num) : n], false, n, nil + } + return nil, false, n, io.EOF +} + +func SkipLengthEncodedString(data []byte) (int, error) { + num, _, n, err := LengthEncodedInt(data) + if err != nil{ + return 0, err + } + if num < 1 { + return n, nil + } + + n += int(num) + + if len(data) >= n { + return n, nil + } + return n, io.EOF +} + +// PutLengthEncodedInt https://dev.mysql.com/doc/internals/en/integer.html#packet-Protocol::LengthEncodedInteger +func PutLengthEncodedInt(n uint64) []byte { + switch { + case n <= 250: + return []byte{byte(n)} + + case n <= 0xffff: + return []byte{0xfc, byte(n), byte(n >> 8)} + + case n <= 0xffffff: + return []byte{0xfd, byte(n), byte(n >> 8), byte(n >> 16)} + + case n <= 0xffffffffffffffff: + return []byte{0xfe, byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24), + byte(n >> 32), byte(n >> 40), byte(n >> 48), byte(n >> 56)} + } + return nil +} + +// PutLengthEncodedString https://dev.mysql.com/doc/internals/en/string.html#packet-Protocol::LengthEncodedString +func PutLengthEncodedString(b []byte) []byte { + data := make([]byte, 0, len(b)+9) + data = append(data, PutLengthEncodedInt(uint64(len(b)))...) + data = append(data, b...) + return data +} + +func Uint16ToBytes(n uint16) []byte { + return []byte{ + byte(n), + byte(n >> 8), + } +} + +func Uint32ToBytes(n uint32) []byte { + return []byte{ + byte(n), + byte(n >> 8), + byte(n >> 16), + byte(n >> 24), + } +} + +func Uint64ToBytes(n uint64) []byte { + return []byte{ + byte(n), + byte(n >> 8), + byte(n >> 16), + byte(n >> 24), + byte(n >> 32), + byte(n >> 40), + byte(n >> 48), + byte(n >> 56), + } +} diff --git a/decryptor/postgresql/pg_decryptor.go b/decryptor/postgresql/pg_decryptor.go index ffd0e215d..08fbdddb3 100644 --- a/decryptor/postgresql/pg_decryptor.go +++ b/decryptor/postgresql/pg_decryptor.go @@ -19,15 +19,17 @@ import ( "crypto/tls" "encoding/binary" "errors" + "io" + "net" + "time" + "github.com/cossacklabs/acra/decryptor/base" acra_io "github.com/cossacklabs/acra/io" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/network" "github.com/cossacklabs/acra/utils" "github.com/cossacklabs/acra/zone" log "github.com/sirupsen/logrus" - "io" - "net" - "time" ) type DataRow struct { @@ -106,17 +108,17 @@ func (row *DataRow) UpdateColumnAndDataSize(oldColumnLength, newColumnLength int return true } // something was decrypted and size should be less that was before - log.Debugf("modify response size: %v -> %v", oldColumnLength, newColumnLength) + log.Debugf("Modify response size: %v -> %v", oldColumnLength, newColumnLength) // update column data size sizeDiff := oldColumnLength - newColumnLength - log.Debugf("old column size: %v; New column size: %v", oldColumnLength, newColumnLength) + log.Debugf("Old column size: %v; New column size: %v", oldColumnLength, newColumnLength) if newColumnLength > oldColumnLength { row.errCh <- errors.New("decrypted size is more than encrypted") return false } binary.BigEndian.PutUint32(row.columnSizePointer, uint32(newColumnLength)) - log.Debugf("old data size: %v; new data size: %v", row.dataLength, row.dataLength-sizeDiff) + log.Debugf("Old data size: %v; new data size: %v", row.dataLength, row.dataLength-sizeDiff) // update data row size row.dataLength -= sizeDiff row.SetDataSize(row.dataLength) @@ -124,7 +126,7 @@ func (row *DataRow) UpdateColumnAndDataSize(oldColumnLength, newColumnLength int } func (row *DataRow) ReadDataLength() bool { - log.Debugln("read data length") + log.Debugln("Read data length") // read full data row length n, err := row.reader.Read(row.output[:DATA_ROW_LENGTH_BUF_SIZE]) if !base.CheckReadWrite(n, DATA_ROW_LENGTH_BUF_SIZE, err, row.errCh) { @@ -155,19 +157,7 @@ func (row *DataRow) Flush() bool { return true } -type PgDecryptorConfig struct { - serverKeyPath string - serverCertPath string -} - -func NewPgDecryptorConfig(tlsKeyPath, tlsCertPath string) (*PgDecryptorConfig, error) { - return &PgDecryptorConfig{serverKeyPath: tlsKeyPath, serverCertPath: tlsCertPath}, nil -} -func (config *PgDecryptorConfig) getCertificate() (tls.Certificate, error) { - return tls.LoadX509KeyPair(config.serverCertPath, config.serverKeyPath) -} - -func PgDecryptStream(decryptor base.Decryptor, config *PgDecryptorConfig, dbConnection net.Conn, clientConnection net.Conn, errCh chan<- error) { +func PgDecryptStream(decryptor base.Decryptor, tlsConfig *tls.Config, dbConnection net.Conn, clientConnection net.Conn, errCh chan<- error) { writer := bufio.NewWriter(clientConnection) reader := acra_io.NewExtendedBufferedReader(bufio.NewReader(dbConnection)) @@ -193,46 +183,45 @@ func PgDecryptStream(decryptor base.Decryptor, config *PgDecryptorConfig, dbConn writer.Flush() continue } else if row.buf[0] == 'S' { - log.Debugln("start tls proxy") - cer, err := config.getCertificate() - if err != nil { - errCh <- err - log.Println(err) - return - } + log.Debugln("Start tls proxy") // stop reading from client in goroutine - if err = clientConnection.SetDeadline(time.Now()); err != nil { - log.WithError(err).Error("can't set deadline") + if err := clientConnection.SetDeadline(time.Now()); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantSetDeadlineToClientConnection). + Errorln("Can't set deadline") errCh <- err return } + // TODO: refactor it to avoid using sleep // back control and allow golang runtime handle deadline in background goroutine time.Sleep(time.Millisecond) // reset deadline - if err = clientConnection.SetDeadline(time.Time{}); err != nil { - log.WithError(err).Error("can't set deadline") + if err := clientConnection.SetDeadline(time.Time{}); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantSetDeadlineToClientConnection). + Errorln("Can't set deadline") errCh <- err return } log.Debugln("init tls with client") // convert to tls connection - tlsClientConnection := tls.Server(clientConnection, &tls.Config{Certificates: []tls.Certificate{cer}}) - if err = writer.Flush(); err != nil { - log.WithError(err).Error("can't flush writer") + tlsClientConnection := tls.Server(clientConnection, tlsConfig) + if err := writer.Flush(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantInitializeTLS). + Errorln("Can't flush writer") errCh <- err return } - err = tlsClientConnection.Handshake() - if err != nil { - log.WithError(err).Error("can't initialize tls connection with client") + if err := tlsClientConnection.Handshake(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantInitializeTLS). + Errorln("Can't initialize tls connection with client") errCh <- err return } - log.Debugln("init tls with db") - dbTLSConnection := tls.Client(dbConnection, &tls.Config{InsecureSkipVerify: true}) - if err = dbTLSConnection.Handshake(); err != nil { - log.WithError(err).Println("can't initialize tls connection with db") + log.Debugln("Init tls with db") + dbTLSConnection := tls.Client(dbConnection, tlsConfig) + if err := dbTLSConnection.Handshake(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorDecryptorCantInitializeTLS). + Errorln("Can't initialize tls connection with db") errCh <- err return } @@ -316,7 +305,7 @@ func PgDecryptStream(decryptor base.Decryptor, config *PgDecryptorConfig, dbConn // poison record check // check only if has any action on detection if decryptor.GetPoisonCallbackStorage().HasCallbacks() { - log.Debugln("check poison records") + log.Debugln("Check poison records") block, err := decryptor.SkipBeginInBlock(row.output[row.writeIndex : row.writeIndex+columnDataLength]) if err == nil { poisoned, err := decryptor.CheckPoisonRecord(bytes.NewReader(block)) @@ -355,7 +344,7 @@ func PgDecryptStream(decryptor base.Decryptor, config *PgDecryptorConfig, dbConn // check poison records if decryptor.GetPoisonCallbackStorage().HasCallbacks() { - log.Debugln("check poison records") + log.Debugln("Check poison records") for { beginTagIndex, tagLength := decryptor.BeginTagIndex(row.output[currentIndex:endIndex]) if beginTagIndex == utils.NOT_FOUND { diff --git a/decryptor/postgresql/pg_general_decryptor.go b/decryptor/postgresql/pg_general_decryptor.go index 1091b265b..e0d2fcb94 100644 --- a/decryptor/postgresql/pg_general_decryptor.go +++ b/decryptor/postgresql/pg_general_decryptor.go @@ -63,6 +63,10 @@ func (decryptor *PgDecryptor) SetZoneMatcher(zoneMatcher *zone.ZoneIdMatcher) { decryptor.zoneMatcher = zoneMatcher } +func (decryptor *PgDecryptor) GetZoneMatcher() *zone.ZoneIdMatcher { + return decryptor.zoneMatcher +} + func (decryptor *PgDecryptor) IsMatchedZone() bool { return decryptor.zoneMatcher.IsMatched() && decryptor.keyStore.HasZonePrivateKey(decryptor.zoneMatcher.GetZoneId()) } @@ -172,6 +176,9 @@ func (decryptor *PgDecryptor) GetPrivateKey() (*keys.PrivateKey, error) { } func (decryptor *PgDecryptor) GetPoisonCallbackStorage() *base.PoisonCallbackStorage { + if decryptor.callbackStorage == nil { + decryptor.callbackStorage = base.NewPoisonCallbackStorage() + } return decryptor.callbackStorage } diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..0372faa02 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,110 @@ +# docker + + * `acra-build.dockerfile` - intermediate image for compile all acra components + * `acraserver.dockerfile` - resulting image with acraserver + * `acraproxy.dockerfile` - resulting image with acraproxy + * `acra_configui.dockerfile` - resulting image with acra_configui component + * `acra_genkeys.dockerfile` - resulting image with acra_genkeys tool + * `acra_genauth.dockerfile` - resulting image with acra_genauth tool + * `postgresql-ssl.dockerfile` - Postgresql server container with example SSL + certificates (located at ssl/postgresql directory) + +## Build containers + +```bash +make docker +``` + +# docker-compose + +## Requirements + +Our docker-compose files were created using v3 compose file format. Please check +your docker engine and docker-compose versions with [docker official +compatibility table](https://docs.docker.com/compose/compose-file/compose-versioning/#compatibility-matrix). + +## Configurations + +There are examples with different interconnection types (`client` is not +included into composes and is given only to indicate its position): + + * `docker/docker-compose.pgsql-nossl-server-ssession-proxy.yml` + pgsql <-> acraserver <-SecureSession-> acraproxy <---> client + '-> acra_configui + * `docker/docker-compose.pgsql-nossl-server-ssession-proxy_zonemode.yml` + pgsql <-> acraserver <-SecureSession-> acraproxy <---> client in zone mode + '-> acra_configui + * `docker/docker-compose.pgsql-nossl-server-ssl-proxy.yml` + pgsql <-> acraserver <-SSL-> acraproxy <-SSL-> client + * `docker/docker-compose.pgsql-nossl-server-ssl-proxy_zonemode.yml` + pgsql <-> acraserver <-SSL-> acraproxy <-SSL-> client in zone mode + * `docker/docker-compose.pgsql-ssl-server-ssl-proxy.yml` + pgsql <-SSL-> acraserver <-SSL-> acraproxy <-SSL-> client + * `docker/docker-compose.pgsql-ssl-server-ssl_zonemode.yml` + pgsql <-SSL-> acraserver <-SSL-> client in zone mode + + +## Quick launch + +INSECURE, TEST ONLY! +```bash +docker-compose -f docker/.yml up +``` +This will create `docker/.acrakeys` directory structure, generate all key pairs, +put them to appropriate services' directories and launch all components. + +Now you can connect to: + * 9494/tcp (acraproxy) + * 8000/tcp (acra_configui) in configurations with acraproxy + * 5432/tcp (postgresql) + + +## Normal launch + +Docker containers with names `acra_genkeys_*` and `acra_genauth` were added to +docker-compose files for architecture demonstration and quick start purposes +only. You should remove them from selected compose file, generate and place all +keys manually. + +Please specify ACRA_MASTER_KEY: +```bash +export ACRA_MASTER_KEY=$(echo -n "My_Very_Long_Key_Phrase_ge_32_chars" | base64) +``` + +Also you probably want to define client id +```bash +export ACRA_CLIENT_ID="MyClientID" +``` + +Optionally you may specify docker image tag, which can be one of: + * `stable` or `latest` - stable branch, recommended, default + * `master` or `current` - master branch of github repository + * `` - specify the exact commit in repository + * `` - choose version tag +```bash +# Examples: +# branch +export ACRA_DOCKER_IMAGE_TAG="master" +# commit tag +export ACRA_DOCKER_IMAGE_TAG="2d2348f440aa0c20b20cd23c49dd34eb0d42d6a5" +# version +export ACRA_DOCKER_IMAGE_TAG="0.76-33-g8b16bc2" +``` + +Please define database name and user credentials: +``` +export POSTGRES_DB="" +export POSTGRES_USER="" +export POSTGRES_PASSWORD="" +``` + +For access to acra_configui HTTP interface you can define: +``` +export ACRA_HTTPAUTH_USER= +export ACRA_HTTPAUTH_PASSWORD= +``` + +Now you can run docker-compose: +```bash + docker-compose -f docker/ up +``` diff --git a/docker/acra-build.dockerfile b/docker/acra-build.dockerfile new file mode 100644 index 000000000..a55429bef --- /dev/null +++ b/docker/acra-build.dockerfile @@ -0,0 +1,64 @@ +FROM debian:stretch +# Product version +ARG VERSION +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata +LABEL com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version="$VERSION" \ + com.cossacklabs.product.vcs-ref="$VCS_REF" \ + com.cossacklabs.product.vcs-branch="$VCS_BRANCH" \ + com.cossacklabs.product.component="acraserver" \ + com.cossacklabs.docker.container.build-date="$BUILD_DATE" \ + com.cossacklabs.docker.container.type="build" +# Install dependencies +RUN apt-get update && apt-get -y install \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + git \ + gnupg \ + libssl-dev \ + openssl \ + rsync \ + wget +WORKDIR /root +# Install libthemis, keep sources for later use +RUN ["/bin/bash", "-c", \ + "set -o pipefail && \ + curl -sSL https://pkgs.cossacklabs.com/scripts/libthemis_install.sh | \ + bash -s -- --yes --method source --branch $VCS_BRANCH \ + --without-packing --without-clean"] +# Install golang and set environment variables +RUN GO_SRC_FILE="go1.9.3.linux-amd64.tar.gz" && \ + wget --no-verbose --no-check-certificate \ + "https://storage.googleapis.com/golang/${GO_SRC_FILE}" && \ + tar xf "./${GO_SRC_FILE}" +ENV GOROOT="/root/go" GOPATH="/root/gopath" +ENV PATH="$GOROOT/bin/:$PATH" +ENV GOPATH_ACRA="${GOPATH}/src/github.com/cossacklabs/acra" +COPY ./ "${GOPATH}/src/github.com/cossacklabs/acra/" +RUN mkdir -p "${GOPATH}/src/github.com/cossacklabs/themis/gothemis" && \ + rsync -au themis/gothemis/ \ + "${GOPATH}/src/github.com/cossacklabs/themis/gothemis" +# Fetch and build dependencies +RUN go get -d -t -v -x github.com/cossacklabs/acra/... +# Build previously fetched acra +RUN go get -v -x github.com/cossacklabs/acra/... +# Include script for finding dependencies and prepare resulting directories +COPY docker/collect_dependencies.sh . +RUN chmod +x ./collect_dependencies.sh +# Copy each product and its dependencies to resulting directories +RUN for component in server proxy _genkeys _configui _genauth; do \ + ./collect_dependencies.sh \ + "${GOPATH}/bin/acra${component}" "/container.acra${component}" && \ + cp "${GOPATH}/bin/acra${component}" "/container.acra${component}/"; \ + done +# Copy static resources for acra_configui +RUN cp -r "${GOPATH}/src/github.com/cossacklabs/acra/cmd/acra_configui/static" \ + "/container.acra_configui/" diff --git a/docker/acra_configui.dockerfile b/docker/acra_configui.dockerfile new file mode 100644 index 000000000..c4e469c6f --- /dev/null +++ b/docker/acra_configui.dockerfile @@ -0,0 +1,37 @@ +# Create internal synonym for previuosly built image +ARG VCS_REF +FROM cossacklabs/acra-build:${VCS_REF} as acra-build + +# Build resulting image from scratch +FROM scratch +# Product version +ARG VERSION +# Link to the product repository +ARG VCS_URL +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata, additionally use label-schema namespace +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Cossack Labs" \ + org.label-schema.url="https://cossacklabs.com" \ + org.label-schema.name="Acra web configurator" \ + org.label-schema.description="Acra helps you easily secure your databases in distributed, microservice-rich environments" \ + org.label-schema.version=$VERSION \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.build-date=$BUILD_DATE \ + com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version=$VERSION \ + com.cossacklabs.product.vcs-ref=$VCS_REF \ + com.cossacklabs.product.vcs-branch=$VCS_BRANCH \ + com.cossacklabs.product.component="acra_configui" \ + com.cossacklabs.docker.container.build-date=$BUILD_DATE \ + com.cossacklabs.docker.container.type="product" +# Copy prepared component's folder from acra-build image +COPY --from=acra-build /container.acra_configui/ / +# Base command +ENTRYPOINT ["/acra_configui", "-static_path=/static"] diff --git a/docker/acra_genauth.dockerfile b/docker/acra_genauth.dockerfile new file mode 100644 index 000000000..67573da34 --- /dev/null +++ b/docker/acra_genauth.dockerfile @@ -0,0 +1,40 @@ +# Create internal synonym for previuosly built image +ARG VCS_REF +FROM cossacklabs/acra-build:${VCS_REF} as acra-build + +# Build resulting image from scratch +FROM scratch +# Product version +ARG VERSION +# Link to the product repository +ARG VCS_URL +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata, additionally use label-schema namespace +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Cossack Labs" \ + org.label-schema.url="https://cossacklabs.com" \ + org.label-schema.name="Acra HTTP auth key generator" \ + org.label-schema.description="Acra helps you easily secure your databases in distributed, microservice-rich environments" \ + org.label-schema.version=$VERSION \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.build-date=$BUILD_DATE \ + com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version=$VERSION \ + com.cossacklabs.product.vcs-ref=$VCS_REF \ + com.cossacklabs.product.vcs-branch=$VCS_BRANCH \ + com.cossacklabs.product.component="acra_genauth" \ + com.cossacklabs.docker.container.build-date=$BUILD_DATE \ + com.cossacklabs.docker.container.type="product" +# Copy prepared component's folder from acra-build image +COPY --from=acra-build /container.acra_genauth/ / +VOLUME ["/auth"] +# Base command +ENTRYPOINT ["/acra_genauth"] +# Optional arguments +CMD ["--keys_dir=/auth", "--file=/auth/auth.keys"] diff --git a/docker/acra_genkeys.dockerfile b/docker/acra_genkeys.dockerfile new file mode 100644 index 000000000..f452582c1 --- /dev/null +++ b/docker/acra_genkeys.dockerfile @@ -0,0 +1,40 @@ +# Create internal synonym for previuosly built image +ARG VCS_REF +FROM cossacklabs/acra-build:${VCS_REF} as acra-build + +# Build resulting image from scratch +FROM scratch +# Product version +ARG VERSION +# Link to the product repository +ARG VCS_URL +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata, additionally use label-schema namespace +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Cossack Labs" \ + org.label-schema.url="https://cossacklabs.com" \ + org.label-schema.name="Acra key generator" \ + org.label-schema.description="Acra helps you easily secure your databases in distributed, microservice-rich environments" \ + org.label-schema.version=$VERSION \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.build-date=$BUILD_DATE \ + com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version=$VERSION \ + com.cossacklabs.product.vcs-ref=$VCS_REF \ + com.cossacklabs.product.vcs-branch=$VCS_BRANCH \ + com.cossacklabs.product.component="acra_genkeys" \ + com.cossacklabs.docker.container.build-date=$BUILD_DATE \ + com.cossacklabs.docker.container.type="product" +# Copy prepared component's folder from acra-build image +COPY --from=acra-build /container.acra_genkeys/ / +VOLUME ["/keys"] +# Base command +ENTRYPOINT ["/acra_genkeys"] +# Optional arguments +CMD ["--output=/keys", "--client_id=testclientid"] diff --git a/docker/acraproxy.dockerfile b/docker/acraproxy.dockerfile index 69556c2c4..41b239c9b 100644 --- a/docker/acraproxy.dockerfile +++ b/docker/acraproxy.dockerfile @@ -1,21 +1,41 @@ -FROM golang -RUN apt-get update && apt-get install -y libssl-dev - -# install themis -RUN git clone https://github.com/cossacklabs/themis.git /themis -WORKDIR /themis -RUN make install && ldconfig -RUN rm -rf /themis - -ENV GOPATH /go -WORKDIR /go - -# build acraproxy -RUN go get github.com/cossacklabs/acra/... -RUN go build github.com/cossacklabs/acra/cmd/acraproxy +# Create internal synonym for previuosly built image +ARG VCS_REF +FROM cossacklabs/acra-build:${VCS_REF} as acra-build +# Build resulting image from scratch +FROM scratch +# Product version +ARG VERSION +# Link to the product repository +ARG VCS_URL +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata, additionally use label-schema namespace +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Cossack Labs" \ + org.label-schema.url="https://cossacklabs.com" \ + org.label-schema.name="Acra server" \ + org.label-schema.description="Acra helps you easily secure your databases in distributed, microservice-rich environments" \ + org.label-schema.version=$VERSION \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.build-date=$BUILD_DATE \ + com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version=$VERSION \ + com.cossacklabs.product.vcs-ref=$VCS_REF \ + com.cossacklabs.product.vcs-branch=$VCS_BRANCH \ + com.cossacklabs.product.component="acraproxy" \ + com.cossacklabs.docker.container.build-date=$BUILD_DATE \ + com.cossacklabs.docker.container.type="product" +# Copy prepared component's folder from acra-build image +COPY --from=acra-build /container.acraproxy/ / VOLUME ["/keys"] -ENTRYPOINT ["acraproxy", "--acra_host=acraserver_link", "-v", "--keys_dir=/keys"] - -EXPOSE 9494 -EXPOSE 9191 +EXPOSE 9191 9494 +# Base command +ENTRYPOINT ["/acraproxy"] +# Optional arguments +CMD ["--acra_host=acraserver_link", "-v", "--keys_dir=/keys"] diff --git a/docker/acraserver.dockerfile b/docker/acraserver.dockerfile index 34df7a3e2..6cbc9466e 100644 --- a/docker/acraserver.dockerfile +++ b/docker/acraserver.dockerfile @@ -1,27 +1,41 @@ -FROM golang -RUN apt-get update && apt-get install -y libssl-dev - -# install themis -RUN git clone https://github.com/cossacklabs/themis.git /themis -WORKDIR /themis -RUN make install && ldconfig -RUN rm -rf /themis - -WORKDIR /go -ENV GOPATH /go - -# build acraserver -RUN go get github.com/cossacklabs/acra/... -RUN go build github.com/cossacklabs/acra/cmd/acraserver -RUN go build github.com/cossacklabs/acra/cmd/acra_addzone -RUN go build github.com/cossacklabs/acra/cmd/acra_genpoisonrecord -RUN go build github.com/cossacklabs/acra/cmd/acra_rollback -RUN go build github.com/cossacklabs/acra/cmd/acra_genkeys +# Create internal synonym for previuosly built image +ARG VCS_REF +FROM cossacklabs/acra-build:${VCS_REF} as acra-build +# Build resulting image from scratch +FROM scratch +# Product version +ARG VERSION +# Link to the product repository +ARG VCS_URL +# Hash of the commit +ARG VCS_REF +# Repository branch +ARG VCS_BRANCH +# Date of the build +ARG BUILD_DATE +# Include metadata, additionally use label-schema namespace +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Cossack Labs" \ + org.label-schema.url="https://cossacklabs.com" \ + org.label-schema.name="Acra server" \ + org.label-schema.description="Acra helps you easily secure your databases in distributed, microservice-rich environments" \ + org.label-schema.version=$VERSION \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.build-date=$BUILD_DATE \ + com.cossacklabs.product.name="acra" \ + com.cossacklabs.product.version=$VERSION \ + com.cossacklabs.product.vcs-ref=$VCS_REF \ + com.cossacklabs.product.vcs-branch=$VCS_BRANCH \ + com.cossacklabs.product.component="acraserver" \ + com.cossacklabs.docker.container.build-date=$BUILD_DATE \ + com.cossacklabs.docker.container.type="product" +# Copy prepared component's folder from acra-build image +COPY --from=acra-build /container.acraserver/ / VOLUME ["/keys"] -ENTRYPOINT ["acraserver", "--db_host=postgresql_link", "-v", "--keys_dir=/keys"] - -# acra server port -EXPOSE 9393 -# acra http api port -EXPOSE 9090 +EXPOSE 9090 9393 +# Base command +ENTRYPOINT ["/acraserver"] +# Optional arguments +CMD ["--db_host=postgres", "-v", "--keys_dir=/keys"] diff --git a/docker/collect_dependencies.sh b/docker/collect_dependencies.sh new file mode 100644 index 000000000..4fe6b1207 --- /dev/null +++ b/docker/collect_dependencies.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +FILE_ELF="$1" +DIR_CONTAINER="$2" + +mkdir "$DIR_CONTAINER" + +mapfile -t libs < <(ldd "$FILE_ELF" | grep '=>' | awk '{print $3}') +libs+=($(readelf -l "$FILE_ELF" | grep -Po "(?<=preter:\\s).+(?=\\])")) + +for l in "${libs[@]}"; do + mkdir -p "${DIR_CONTAINER}/$(dirname ${l})" + cp -L "$l" "${DIR_CONTAINER}/${l}" +done diff --git a/docker/docker-compose-with-zones.yml b/docker/docker-compose-with-zones.yml deleted file mode 100644 index f6fb60dc5..000000000 --- a/docker/docker-compose-with-zones.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: "2" - -services: - pg: - image: postgres - ports: - - "5432:5432" - - acraserver: - image: cossacklabs/acraserver - depends_on: - - pg - links: - - pg:postgresql_link - - volumes: - - ../.acrakeys:/keys - - command: ["--zonemode"] - - acraproxy: - image: cossacklabs/acraproxy - depends_on: - - acraserver - ports: - - "9494:9494" - - "9191:9191" - links: - - acraserver:acraserver_link - volumes: - - ../.acrakeys:/keys - - command: ["--client_id=client", "--zonemode"] diff --git a/docker/docker-compose.pgsql-nossl-server-ssession-proxy.yml b/docker/docker-compose.pgsql-nossl-server-ssession-proxy.yml new file mode 100644 index 000000000..157ed44c9 --- /dev/null +++ b/docker/docker-compose.pgsql-nossl-server-ssession-proxy.yml @@ -0,0 +1,186 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_server + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID}_server.pub + acra_genkeys_server: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraserver + --output=/keys/acraserver + --output_public=/keys/acraproxy + # Create keys: + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID} + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}.pub + acra_genkeys_proxy: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraproxy + --output=/keys/acraproxy + --output_public=/keys/acraserver + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Create file with accounts for HTTP access and key for decrypt it + # - ./.acrakeys/acraserver/httpauth.accounts + # - ./.acrakeys/acraserver/auth_key + acra_genauth: + image: "cossacklabs/acra_genauth:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --set + --user=${ACRA_HTTPAUTH_USER:-test} + --password=${ACRA_HTTPAUTH_PASSWORD:-test} + --file=/keys/acraserver/httpauth.accounts + --keys_dir=/keys/acraserver/ + + # Postgresql container + postgresql: + image: postgres + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # Restart server after correct termination, for example after the config + # was changed through the API + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acra_genkeys_writer + - acra_genauth + - postgresql + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraserver-postgresql' - for acraserver and DB interconnection + # - 'acraproxy-acraserver' - for acraserver and acraproxy interconnection + networks: + - acraproxy-acraserver + - acraserver-postgresql + volumes: + # Mount the directory with only the keys for this service. Must be + # rewriteable in case of using API, otherwise should be read-only. + - ./.acrakeys/acraserver:/keys + # Directory with configuration, rewriteable + - ./.acraconfigs/acraserver:/config + command: >- + --db_host=postgresql + --keys_dir=/keys + --auth_keys=/keys/httpauth.accounts + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9090 + --config=/config/acraserver.yaml + -v + + acraproxy: + image: "cossacklabs/acraproxy:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acraserver + # Open the port outside for client application + ports: + - "9494:9494" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraproxy-acraserver' - for interconnection with acraserver + # - 'acraconfigui-acraproxy' - for interconnection with acra_configui + # and external network 'world' for port exposing + networks: + - acraproxy-acraserver + - acraconfigui-acraproxy + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraproxy:/keys:ro + command: >- + --acra_host=acraserver + --keys_dir=/keys + --client_id=${ACRA_CLIENT_ID:-testclientid} + --connection_string=tcp://0.0.0.0:9494 + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9191 + -v + + # Optional lightweight HTTP web server for managing AcraServer's + # certain configuration options + acra_configui: + image: "cossacklabs/acra_configui:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: on-failure + depends_on: + - acraproxy + # Open the port outside for client application + ports: + - "8000:8000" + # We use internal 'acraconfigui-acraproxy' network for acraproxy and + # acra_configui interconnection and external network 'world' for + # port exposing + networks: + - acraconfigui-acraproxy + - world + command: >- + --acra_host=acraproxy + --host=0.0.0.0 + +networks: + world: + acraproxy-acraserver: + internal: true + acraserver-postgresql: + internal: true + acraconfigui-acraproxy: + internal: true diff --git a/docker/docker-compose.pgsql-nossl-server-ssession-proxy_zonemode.yml b/docker/docker-compose.pgsql-nossl-server-ssession-proxy_zonemode.yml new file mode 100644 index 000000000..e58c98bb8 --- /dev/null +++ b/docker/docker-compose.pgsql-nossl-server-ssession-proxy_zonemode.yml @@ -0,0 +1,188 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_server + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID}_server.pub + acra_genkeys_server: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraserver + --output=/keys/acraserver + --output_public=/keys/acraproxy + # Create keys: + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID} + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}.pub + acra_genkeys_proxy: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraproxy + --output=/keys/acraproxy + --output_public=/keys/acraserver + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Create file with accounts for HTTP access and key for decrypt it + # - ./.acrakeys/acraserver/httpauth.accounts + # - ./.acrakeys/acraserver/auth_key + acra_genauth: + image: "cossacklabs/acra_genauth:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --set + --user=${ACRA_HTTPAUTH_USER:-test} + --password=${ACRA_HTTPAUTH_PASSWORD:-test} + --file=/keys/acraserver/httpauth.accounts + --keys_dir=/keys/acraserver/ + + # Postgresql container + postgresql: + image: postgres + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # Restart server after correct termination, for example after the config + # was changed through the API + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acra_genkeys_writer + - acra_genauth + - postgresql + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraserver-postgresql' - for acraserver and DB interconnection + # - 'acraproxy-acraserver' - for acraserver and acraproxy interconnection + networks: + - acraproxy-acraserver + - acraserver-postgresql + volumes: + # Mount the directory with only the keys for this service. Must be + # rewriteable in case of using API, otherwise should be read-only. + - ./.acrakeys/acraserver:/keys + # Directory with configuration, rewriteable + - ./.acraconfigs/acraserver:/config + command: >- + --zonemode + --db_host=postgresql + --keys_dir=/keys + --auth_keys=/keys/httpauth.accounts + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9090 + --config=/config/acraserver.yaml + -v + + acraproxy: + image: "cossacklabs/acraproxy:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acraserver + # Open the port outside for client application and API + ports: + - "9191:9191" + - "9494:9494" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraproxy-acraserver' - for interconnection with acraserver + # - 'acraconfigui-acraproxy' - for interconnection with acra_configui + # and external network 'world' for port exposing + networks: + - acraproxy-acraserver + - acraconfigui-acraproxy + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraproxy:/keys:ro + command: >- + --acra_host=acraserver + --keys_dir=/keys + --client_id=${ACRA_CLIENT_ID:-testclientid} + --enable_http_api + --connection_string=tcp://0.0.0.0:9494 + --connection_api_string=tcp://0.0.0.0:9191 + -v + + # Optional lightweight HTTP web server for managing AcraServer's + # certain configuration options + acra_configui: + image: "cossacklabs/acra_configui:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: on-failure + depends_on: + - acraproxy + # Open the port outside for client application + ports: + - "8000:8000" + # We use internal 'acraconfigui-acraproxy' network for acraproxy and + # acra_configui interconnection and external network 'world' for + # port exposing + networks: + - acraconfigui-acraproxy + - world + command: >- + --acra_host=acraproxy + --host=0.0.0.0 + +networks: + world: + acraproxy-acraserver: + internal: true + acraserver-postgresql: + internal: true + acraconfigui-acraproxy: + internal: true diff --git a/docker/docker-compose.pgsql-nossl-server-ssl-proxy.yml b/docker/docker-compose.pgsql-nossl-server-ssl-proxy.yml new file mode 100644 index 000000000..2ba82584e --- /dev/null +++ b/docker/docker-compose.pgsql-nossl-server-ssl-proxy.yml @@ -0,0 +1,161 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_server + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID}_server.pub + acra_genkeys_server: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraserver + --output=/keys/acraserver + --output_public=/keys/acraproxy + # Create keys: + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID} + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}.pub + acra_genkeys_proxy: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraproxy + --output=/keys/acraproxy + --output_public=/keys/acraserver + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Postgresql container + postgresql: + image: postgres + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # Restart server after correct termination, for example after the config + # was changed through the API + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acra_genkeys_writer + - postgresql + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraserver-postgresql' - for acraserver and DB interconnection + # - 'acraproxy-acraserver' - for acraserver and acraproxy interconnection + networks: + - acraproxy-acraserver + - acraserver-postgresql + volumes: + # Mount the directory with only the keys for this service. Must be + # rewriteable in case of using API, otherwise should be read-only. + - ./.acrakeys/acraserver:/keys + # Directory with configuration, rewriteable + - ./.acraconfigs/acraserver:/config + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraserver:/ssl.server:ro + command: >- + --db_host=postgresql + --keys_dir=/keys + --auth_keys=/keys/httpauth.accounts + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9090 + --config=/config/acraserver.yaml + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.server/acraserver.crt + --tls_key=/ssl.server/acraserver.key + --no_encryption + -v + + acraproxy: + image: "cossacklabs/acraproxy:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acraserver + # Open the port outside for client application + ports: + - "9494:9494" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraproxy-acraserver' - for interconnection with acraserver + # and external network 'world' for port exposing + networks: + - acraproxy-acraserver + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraproxy:/keys:ro + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraproxy:/ssl.proxy:ro + command: >- + --acra_host=acraserver + --keys_dir=/keys + --client_id=${ACRA_CLIENT_ID:-testclientid} + --connection_string=tcp://0.0.0.0:9494 + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9191 + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.proxy/acraproxy.crt + --tls_key=/ssl.proxy/acraproxy.key + --tls_sni=acraserver + --no_encryption + -v + +networks: + world: + acraproxy-acraserver: + internal: true + acraserver-postgresql: + internal: true diff --git a/docker/docker-compose.pgsql-nossl-server-ssl-proxy_zonemode.yml b/docker/docker-compose.pgsql-nossl-server-ssl-proxy_zonemode.yml new file mode 100644 index 000000000..291693312 --- /dev/null +++ b/docker/docker-compose.pgsql-nossl-server-ssl-proxy_zonemode.yml @@ -0,0 +1,162 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_server + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID}_server.pub + acra_genkeys_server: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraserver + --output=/keys/acraserver + --output_public=/keys/acraproxy + # Create keys: + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID} + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}.pub + acra_genkeys_proxy: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraproxy + --output=/keys/acraproxy + --output_public=/keys/acraserver + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Postgresql container + postgresql: + image: postgres + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # Restart server after correct termination, for example after the config + # was changed through the API + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acra_genkeys_writer + - postgresql + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraserver-postgresql' - for acraserver and DB interconnection + # - 'acraproxy-acraserver' - for acraserver and acraproxy interconnection + networks: + - acraproxy-acraserver + - acraserver-postgresql + volumes: + # Mount the directory with only the keys for this service. Must be + # rewriteable in case of using API, otherwise should be read-only. + - ./.acrakeys/acraserver:/keys + # Directory with configuration, rewriteable + - ./.acraconfigs/acraserver:/config + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraserver:/ssl.server:ro + command: >- + --zonemode + --db_host=postgresql + --keys_dir=/keys + --auth_keys=/keys/httpauth.accounts + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9090 + --config=/config/acraserver.yaml + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.server/acraserver.crt + --tls_key=/ssl.server/acraserver.key + --no_encryption + -v + + acraproxy: + image: "cossacklabs/acraproxy:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acraserver + # Open the port outside for client application + ports: + - "9494:9494" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraproxy-acraserver' - for interconnection with acraserver + # and external network 'world' for port exposing + networks: + - acraproxy-acraserver + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraproxy:/keys:ro + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraproxy:/ssl.proxy:ro + command: >- + --acra_host=acraserver + --keys_dir=/keys + --client_id=${ACRA_CLIENT_ID:-testclientid} + --connection_string=tcp://0.0.0.0:9494 + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9191 + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.proxy/acraproxy.crt + --tls_key=/ssl.proxy/acraproxy.key + --tls_sni=acraserver + --no_encryption + -v + +networks: + world: + acraproxy-acraserver: + internal: true + acraserver-postgresql: + internal: true diff --git a/docker/docker-compose.pgsql-ssl-server-ssl-proxy.yml b/docker/docker-compose.pgsql-ssl-server-ssl-proxy.yml new file mode 100644 index 000000000..90722db65 --- /dev/null +++ b/docker/docker-compose.pgsql-ssl-server-ssl-proxy.yml @@ -0,0 +1,166 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_server + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID}_server.pub + acra_genkeys_server: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraserver + --output=/keys/acraserver + --output_public=/keys/acraproxy + # Create keys: + # - ./.acrakeys/acraproxy/${ACRA_CLIENT_ID} + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}.pub + acra_genkeys_proxy: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --acraproxy + --output=/keys/acraproxy + --output_public=/keys/acraserver + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + network_mode: "none" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + - ./.acrakeys:/keys + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Postgresql container + postgresql: + # Build and run container based on official postgresql image with + # strict SSL mode + build: + context: ../ + dockerfile: docker/postgresql-ssl.dockerfile + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # Restart server after correct termination, for example after the config + # was changed through the API + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acra_genkeys_writer + - postgresql + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraserver-postgresql' - for acraserver and DB interconnection + # - 'acraproxy-acraserver' - for acraserver and acraproxy interconnection + networks: + - acraproxy-acraserver + - acraserver-postgresql + volumes: + # Mount the directory with only the keys for this service. Must be + # rewriteable in case of using API, otherwise should be read-only. + - ./.acrakeys/acraserver:/keys + # Directory with configuration, rewriteable + - ./.acraconfigs/acraserver:/config + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraserver:/ssl.server:ro + command: >- + --db_host=postgresql + --keys_dir=/keys + --auth_keys=/keys/httpauth.accounts + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9090 + --config=/config/acraserver.yaml + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.server/acraserver.crt + --tls_key=/ssl.server/acraserver.key + --tls_sni=postgresql + --no_encryption + -v + + acraproxy: + image: "cossacklabs/acraproxy:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: always + depends_on: + - acra_genkeys_server + - acra_genkeys_proxy + - acraserver + # Open the port outside for client application + ports: + - "9494:9494" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal networks: + # - 'acraproxy-acraserver' - for interconnection with acraserver + # and external network 'world' for port exposing + networks: + - acraproxy-acraserver + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraproxy:/keys:ro + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraproxy:/ssl.proxy:ro + command: >- + --acra_host=acraserver + --keys_dir=/keys + --client_id=${ACRA_CLIENT_ID:-testclientid} + --connection_string=tcp://0.0.0.0:9494 + --enable_http_api + --connection_api_string=tcp://0.0.0.0:9191 + --tls + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.proxy/acraproxy.crt + --tls_key=/ssl.proxy/acraproxy.key + --tls_sni=acraserver + --no_encryption + -v + +networks: + world: + acraproxy-acraserver: + internal: true + acraserver-postgresql: + internal: true diff --git a/docker/docker-compose.pgsql-ssl-server-ssl_zonemode.yml b/docker/docker-compose.pgsql-ssl-server-ssl_zonemode.yml new file mode 100644 index 000000000..5f49138a4 --- /dev/null +++ b/docker/docker-compose.pgsql-ssl-server-ssl_zonemode.yml @@ -0,0 +1,86 @@ +version: "3" + +services: + # Create keys: + # - ./.acrakeys/acraserver/${ACRA_CLIENT_ID}_storage + # - ./.acrakeys/acrawriter/${ACRA_CLIENT_ID}_storage.pub + acra_genkeys_writer: + # You can specify docker image tag in the environment + # variable ACRA_DOCKER_IMAGE_TAG or run by default with 'latest' images + image: "cossacklabs/acra_genkeys:${ACRA_DOCKER_IMAGE_TAG:-latest}" + # We do not need network for keys' generation at all + network_mode: "none" + environment: + # INSECURE!!! You MUST define your own ACRA_MASTER_KEY + # The default is only for testing purposes + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + volumes: + # Mount the whole ./.acrakeys directory to be able generate keys and + # place them in services' subdirectories + - ./.acrakeys:/keys + # Please specify ACRA_CLIENT_ID environment variable, otherwise run with + # default 'testclientid' client id + command: >- + --client_id=${ACRA_CLIENT_ID:-testclientid} + --storage + --output=/keys/acraserver + --output_public=/keys/acrawriter + + # Postgresql container + postgresql: + # Build and run container based on official postgresql image with + # strict SSL mode + build: + context: ../ + dockerfile: docker/postgresql-ssl.dockerfile + # INSECURE!!! You MUST define your own DB name and credentials + environment: + POSTGRES_DB: ${POSTGRES_DB:-test} + POSTGRES_USER: ${POSTGRES_USER:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-test} + # Open the port outside for writer + ports: + - "5432:5432" + # We use internal 'acraserver-postgresql' network for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + + acraserver: + image: "cossacklabs/acraserver:${ACRA_DOCKER_IMAGE_TAG:-latest}" + restart: on-failure + depends_on: + - acra_genkeys_writer + - postgresql + # Open the port outside for client application + ports: + - "9393:9393" + environment: + ACRA_MASTER_KEY: ${ACRA_MASTER_KEY:-UHZ3VUNNeTJ0SEFhbWVjNkt4eDdVYkc2WnNpUTlYa0E=} + # We use internal network 'acraserver-postgresql' for acraserver and + # DB interconnection and external network 'world' for port exposing + networks: + - acraserver-postgresql + - world + volumes: + # Mount the directory with only the keys for this service + - ./.acrakeys/acraserver:/keys:ro + # Mount directories with SSL certificates + - ./ssl/ca:/ssl.ca:ro + - ./ssl/acraserver:/ssl.server:ro + command: >- + --zonemode + --db_host=postgresql + --keys_dir=/keys + --tls_ca=/ssl.ca/example.cossacklabs.com.CA.crt + --tls_cert=/ssl.server/acraserver.crt + --tls_key=/ssl.server/acraserver.key + --tls_sni=postgresql + --no_encryption + -v + +networks: + world: + acraserver-postgresql: + internal: true diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 6bda1c9d5..000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: "2" - -services: - pg: - image: postgres - ports: - - "5432:5432" - - acraserver: - image: cossacklabs/acraserver - depends_on: - - pg - links: - - pg:postgresql_link - volumes: - - ../.acrakeys:/keys - - acraproxy: - image: cossacklabs/acraproxy - depends_on: - - acraserver - ports: - - "9494:9494" - links: - - acraserver:acraserver_link - volumes: - - ../.acrakeys:/keys - command: --client_id=client diff --git a/docker/postgresql-ssl-configure.sh b/docker/postgresql-ssl-configure.sh new file mode 100644 index 000000000..984e3b72d --- /dev/null +++ b/docker/postgresql-ssl-configure.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +for f in root.crt server.crt server.key; do + cp /tmp.ssl/${f} "${PGDATA}/" + chown postgres:postgres "${PGDATA}/${f}" + chmod 0600 "${PGDATA}/${f}" +done + +set_pg_option() { + sed -i "s/^#*${1}\\s*=.*/${1} = ${2}/g" "$PGDATA/postgresql.conf" +} +set_pg_option "listen_addresses" "'*'" +set_pg_option "ssl" "on" +set_pg_option "ssl_ca_file" "'root.crt'" +set_pg_option "ssl_cert_file" "'server.crt'" +set_pg_option "ssl_key_file" "'server.key'" + +# remove all default host* rules +sed -i '/^host.*/d' "$PGDATA/pg_hba.conf" +# configure strict ssl access +echo -e "hostssl\\t${POSTGRES_DB}\\t${POSTGRES_USER}\\t0.0.0.0/0\\tmd5" >> \ + "$PGDATA/pg_hba.conf" diff --git a/docker/postgresql-ssl.dockerfile b/docker/postgresql-ssl.dockerfile new file mode 100644 index 000000000..2a39d3bb4 --- /dev/null +++ b/docker/postgresql-ssl.dockerfile @@ -0,0 +1,9 @@ +FROM postgres:9.6 + +# Original postgresql init script expects empty $PGDATA so we initially place +# certificates into the image to the intermediate directory +COPY docker/ssl/postgresql/postgresql.crt /tmp.ssl/server.crt +COPY docker/ssl/postgresql/postgresql.key /tmp.ssl/server.key +COPY docker/ssl/ca/example.cossacklabs.com.CA.crt /tmp.ssl/root.crt + +COPY docker/postgresql-ssl-configure.sh /docker-entrypoint-initdb.d/ diff --git a/docker/ssl/acraproxy/acraproxy.crt b/docker/ssl/acraproxy/acraproxy.crt new file mode 100644 index 000000000..1a2f5a959 --- /dev/null +++ b/docker/ssl/acraproxy/acraproxy.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICjzCCAhQCCQD79qa2sMF9+zAKBggqhkjOPQQDAjBaMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRMwEQYDVQQDDAphY3Jhc2VydmVyMB4XDTE4MDMwNTE2MDYwMloXDTE5 +MDIyODE2MDYwMlowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJYWNy +YXByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0DLp2iFSu4+f +3tWrKUwwTFMXgHAjo1f+RpnJGn3mAMyQcraSTM23hMz+hYROpLJVIeIsOA9hJcCl +a8y9XbecmcDrCageFjnd0g2/t7Fm8mAuehBRRWYWSovQy4Uq/aLIgn9LxQHk6IT+ +xfrr95bfPurlLrzFoUNQL3qhA28Pdj7gpq3+g8j6yNRdEhgLVeyjg38n63OcgAY8 +XmqWFUIoXANQSANUfrjxVB6Y1t/vzLm8yPs7uEkKHwuNmYwNzFAeBe1VYzRAgfB1 +zqC9Glyn4WRBVw22FJrT5YvSXfKLbYJRwRXB2vOIaXgLlZR2BOY+NPV83MV03iud +05Chzn7eMwIDAQABMAoGCCqGSM49BAMCA2kAMGYCMQCte78tUJ1jOqDLNBSaVjL8 +clmfFIa5jaGUlkmDhdiT23By63WjAzcqY0JmbaSoO0oCMQDorwla+ZZkNyjRqc2r +kH57wMaMd7qBDponA2bSytDP/bKr7dxeLqsyqbnMrlIX4/c= +-----END CERTIFICATE----- diff --git a/docker/ssl/acraproxy/acraproxy.key b/docker/ssl/acraproxy/acraproxy.key new file mode 100644 index 000000000..18770bcbb --- /dev/null +++ b/docker/ssl/acraproxy/acraproxy.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDQMunaIVK7j5/e +1aspTDBMUxeAcCOjV/5GmckafeYAzJBytpJMzbeEzP6FhE6kslUh4iw4D2ElwKVr +zL1dt5yZwOsJqB4WOd3SDb+3sWbyYC56EFFFZhZKi9DLhSr9osiCf0vFAeTohP7F ++uv3lt8+6uUuvMWhQ1AveqEDbw92PuCmrf6DyPrI1F0SGAtV7KODfyfrc5yABjxe +apYVQihcA1BIA1R+uPFUHpjW3+/MubzI+zu4SQofC42ZjA3MUB4F7VVjNECB8HXO +oL0aXKfhZEFXDbYUmtPli9Jd8ottglHBFcHa84hpeAuVlHYE5j409XzcxXTeK53T +kKHOft4zAgMBAAECggEATBDN76wNFgm7VyapikUTwE77XymZW6xicljtaIhm6BPV +EpQxj235hsN+mjlkojelcuO3VCQKUki4J1J+PSdAR8x8EuMhWu15Za0wRmTCP/tz +/5TGnJxXuJSsjC7zLgezSjpH7Ipsn6c3jg3G+IZeuhYH9bNyYSC+wxoCicah14ko +svJHV3NANSORzbqxmZ1wHy29e93eS6dq+O+LBYClVjhw4lSfIrhlx4LmEyXZYQOk +aitxz3Uv2RkemoMi9Yw6R/64BSuPRtFebZkw9ipP/BC4MC+vNH2/rVjxG6ashFqb +qsnkQzl4FsPKl+0ruyccvY4oTj5suWYw2RWU0vjCwQKBgQDuRDB5FmHcf1sd1yvV +is+mYbE1atv17TBn1DKGVcSWk79HcGXhXvt2E1oVsOTVnHqwEffah8tyJXYFsWsP +7a/1GMeAmaPrhd0IkIpEM0DbYGTQ6F1uaLo2IjwNIFzX0TXvf+Dww1YvwppkL4A1 +5I+vOPO066BEM2lfru6WcOrrFwKBgQDfsdVA5aJlcjuA59J4zh3/Iq7DcSkeeXTZ +wO7jeTqRHDKChqRcuK2RJoB0M3P/V+E/X6+DzJlQURLs4d3KM0/Ypr5WElBkLyHT +YT7x2moqsWbjduavdrVgqMovQJSsZLYaLaU3l2zQT1L9k3/ltu5hblVempl7rmfP +eKSh+6snRQKBgHp/eGRoy3tvxsq6u4CYU1X5WABcpiX0AjT/ddJ2+hFoeKkj8l1C +VgpIvMH2JlBkmPc45bLmqgRPmjQnGSIhU5uxV7CYTRxjwFYM6elSaH/hOTPmo1KG +aWY3h6RABTu4BgDSQDXIV+FKLdJgUYxjrDOsFi/oDIfD3uMgru2NtFmVAoGBAMX8 +asf+twZc3aeRBysfG1OWyeF3xbIQQ8j7RzSUNq76qwX1z4G1fwGqdyTh6XgFuvpR +YVIhA00gBMUegCQX2ELkCjC6EucpBCJHvuNmsnLJA0yuDy0bvxsnKZQ675vJo5d1 +8PZMEuYoX0bKhve1OjWH5w1Nfi0GxyDNIcGwsuKVAoGBAOGbDWzsbee4X65+YkwO +ssIzEAsB7vkiSu5Dn2nL0EmBRWQFfOfXPbrYlRHYTs09/Qfm2TZK20kGxoBqiHf3 +Pz8/Nu0517b4J1ssPAu/zJ7kFpUtkoaSUf2MmXRvCi6xqYYtnvFQFGQbjHM2ySTu +OMY3q9GhMTp9A7mLGaDNiaym +-----END PRIVATE KEY----- diff --git a/docker/ssl/acraserver/acraserver.crt b/docker/ssl/acraserver/acraserver.crt new file mode 100644 index 000000000..c0ac64bf8 --- /dev/null +++ b/docker/ssl/acraserver/acraserver.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4DCCAWcCCQDk707vUhDpOjAKBggqhkjOPQQDAjBaMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRMwEQYDVQQDDAphY3Jhc2VydmVyMB4XDTE4MDMwNTE2MDgwMVoXDTI4 +MDMwMjE2MDgwMVowWjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKYWNy +YXNlcnZlcjB2MBAGByqGSM49AgEGBSuBBAAiA2IABB0cgtd3rItFJ4J84KSMhr/d +6q29DT8e2a6+cXIzZhcW/OYLUhltYETamwUqrvgYwKkGf1U2oscyhMmGeltHYFtf +aMCVxcAiyCa+lMJw2pgk9e1qljDsckrPinSVHptHiDAKBggqhkjOPQQDAgNnADBk +AjAkDSYpKd7jLKjaJGbu+/mCBu8I2fKQD/zcVxirtdMBD1Xhghs1hpLiGCpVdDDU +QewCMAjm9O81tKpDTWqeRW93kGRw+05g5XZe68/IRfCo3R9A4+xj1AqzY41/v9Dw +dFPjbg== +-----END CERTIFICATE----- diff --git a/docker/ssl/acraserver/acraserver.key b/docker/ssl/acraserver/acraserver.key new file mode 100644 index 000000000..29f877f6e --- /dev/null +++ b/docker/ssl/acraserver/acraserver.key @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAyKqW1QeWZET8794xQa4JINQJXTOUIt29jKhVnFf/MFBFD6s8y7nS5 +MP8PLzejdrugBwYFK4EEACKhZANiAAQdHILXd6yLRSeCfOCkjIa/3eqtvQ0/Htmu +vnFyM2YXFvzmC1IZbWBE2psFKq74GMCpBn9VNqLHMoTJhnpbR2BbX2jAlcXAIsgm +vpTCcNqYJPXtapYw7HJKz4p0lR6bR4g= +-----END EC PRIVATE KEY----- diff --git a/docker/ssl/acrawriter/acrawriter.crt b/docker/ssl/acrawriter/acrawriter.crt new file mode 100644 index 000000000..1a2f5a959 --- /dev/null +++ b/docker/ssl/acrawriter/acrawriter.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICjzCCAhQCCQD79qa2sMF9+zAKBggqhkjOPQQDAjBaMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRMwEQYDVQQDDAphY3Jhc2VydmVyMB4XDTE4MDMwNTE2MDYwMloXDTE5 +MDIyODE2MDYwMlowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJYWNy +YXByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0DLp2iFSu4+f +3tWrKUwwTFMXgHAjo1f+RpnJGn3mAMyQcraSTM23hMz+hYROpLJVIeIsOA9hJcCl +a8y9XbecmcDrCageFjnd0g2/t7Fm8mAuehBRRWYWSovQy4Uq/aLIgn9LxQHk6IT+ +xfrr95bfPurlLrzFoUNQL3qhA28Pdj7gpq3+g8j6yNRdEhgLVeyjg38n63OcgAY8 +XmqWFUIoXANQSANUfrjxVB6Y1t/vzLm8yPs7uEkKHwuNmYwNzFAeBe1VYzRAgfB1 +zqC9Glyn4WRBVw22FJrT5YvSXfKLbYJRwRXB2vOIaXgLlZR2BOY+NPV83MV03iud +05Chzn7eMwIDAQABMAoGCCqGSM49BAMCA2kAMGYCMQCte78tUJ1jOqDLNBSaVjL8 +clmfFIa5jaGUlkmDhdiT23By63WjAzcqY0JmbaSoO0oCMQDorwla+ZZkNyjRqc2r +kH57wMaMd7qBDponA2bSytDP/bKr7dxeLqsyqbnMrlIX4/c= +-----END CERTIFICATE----- diff --git a/docker/ssl/acrawriter/acrawriter.key b/docker/ssl/acrawriter/acrawriter.key new file mode 100644 index 000000000..18770bcbb --- /dev/null +++ b/docker/ssl/acrawriter/acrawriter.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDQMunaIVK7j5/e +1aspTDBMUxeAcCOjV/5GmckafeYAzJBytpJMzbeEzP6FhE6kslUh4iw4D2ElwKVr +zL1dt5yZwOsJqB4WOd3SDb+3sWbyYC56EFFFZhZKi9DLhSr9osiCf0vFAeTohP7F ++uv3lt8+6uUuvMWhQ1AveqEDbw92PuCmrf6DyPrI1F0SGAtV7KODfyfrc5yABjxe +apYVQihcA1BIA1R+uPFUHpjW3+/MubzI+zu4SQofC42ZjA3MUB4F7VVjNECB8HXO +oL0aXKfhZEFXDbYUmtPli9Jd8ottglHBFcHa84hpeAuVlHYE5j409XzcxXTeK53T +kKHOft4zAgMBAAECggEATBDN76wNFgm7VyapikUTwE77XymZW6xicljtaIhm6BPV +EpQxj235hsN+mjlkojelcuO3VCQKUki4J1J+PSdAR8x8EuMhWu15Za0wRmTCP/tz +/5TGnJxXuJSsjC7zLgezSjpH7Ipsn6c3jg3G+IZeuhYH9bNyYSC+wxoCicah14ko +svJHV3NANSORzbqxmZ1wHy29e93eS6dq+O+LBYClVjhw4lSfIrhlx4LmEyXZYQOk +aitxz3Uv2RkemoMi9Yw6R/64BSuPRtFebZkw9ipP/BC4MC+vNH2/rVjxG6ashFqb +qsnkQzl4FsPKl+0ruyccvY4oTj5suWYw2RWU0vjCwQKBgQDuRDB5FmHcf1sd1yvV +is+mYbE1atv17TBn1DKGVcSWk79HcGXhXvt2E1oVsOTVnHqwEffah8tyJXYFsWsP +7a/1GMeAmaPrhd0IkIpEM0DbYGTQ6F1uaLo2IjwNIFzX0TXvf+Dww1YvwppkL4A1 +5I+vOPO066BEM2lfru6WcOrrFwKBgQDfsdVA5aJlcjuA59J4zh3/Iq7DcSkeeXTZ +wO7jeTqRHDKChqRcuK2RJoB0M3P/V+E/X6+DzJlQURLs4d3KM0/Ypr5WElBkLyHT +YT7x2moqsWbjduavdrVgqMovQJSsZLYaLaU3l2zQT1L9k3/ltu5hblVempl7rmfP +eKSh+6snRQKBgHp/eGRoy3tvxsq6u4CYU1X5WABcpiX0AjT/ddJ2+hFoeKkj8l1C +VgpIvMH2JlBkmPc45bLmqgRPmjQnGSIhU5uxV7CYTRxjwFYM6elSaH/hOTPmo1KG +aWY3h6RABTu4BgDSQDXIV+FKLdJgUYxjrDOsFi/oDIfD3uMgru2NtFmVAoGBAMX8 +asf+twZc3aeRBysfG1OWyeF3xbIQQ8j7RzSUNq76qwX1z4G1fwGqdyTh6XgFuvpR +YVIhA00gBMUegCQX2ELkCjC6EucpBCJHvuNmsnLJA0yuDy0bvxsnKZQ675vJo5d1 +8PZMEuYoX0bKhve1OjWH5w1Nfi0GxyDNIcGwsuKVAoGBAOGbDWzsbee4X65+YkwO +ssIzEAsB7vkiSu5Dn2nL0EmBRWQFfOfXPbrYlRHYTs09/Qfm2TZK20kGxoBqiHf3 +Pz8/Nu0517b4J1ssPAu/zJ7kFpUtkoaSUf2MmXRvCi6xqYYtnvFQFGQbjHM2ySTu +OMY3q9GhMTp9A7mLGaDNiaym +-----END PRIVATE KEY----- diff --git a/docker/ssl/ca/example.cossacklabs.com.CA.crt b/docker/ssl/ca/example.cossacklabs.com.CA.crt new file mode 100644 index 000000000..c0ac64bf8 --- /dev/null +++ b/docker/ssl/ca/example.cossacklabs.com.CA.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4DCCAWcCCQDk707vUhDpOjAKBggqhkjOPQQDAjBaMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRMwEQYDVQQDDAphY3Jhc2VydmVyMB4XDTE4MDMwNTE2MDgwMVoXDTI4 +MDMwMjE2MDgwMVowWjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKYWNy +YXNlcnZlcjB2MBAGByqGSM49AgEGBSuBBAAiA2IABB0cgtd3rItFJ4J84KSMhr/d +6q29DT8e2a6+cXIzZhcW/OYLUhltYETamwUqrvgYwKkGf1U2oscyhMmGeltHYFtf +aMCVxcAiyCa+lMJw2pgk9e1qljDsckrPinSVHptHiDAKBggqhkjOPQQDAgNnADBk +AjAkDSYpKd7jLKjaJGbu+/mCBu8I2fKQD/zcVxirtdMBD1Xhghs1hpLiGCpVdDDU +QewCMAjm9O81tKpDTWqeRW93kGRw+05g5XZe68/IRfCo3R9A4+xj1AqzY41/v9Dw +dFPjbg== +-----END CERTIFICATE----- diff --git a/docker/ssl/postgresql/postgresql.crt b/docker/ssl/postgresql/postgresql.crt new file mode 100644 index 000000000..c0ac64bf8 --- /dev/null +++ b/docker/ssl/postgresql/postgresql.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4DCCAWcCCQDk707vUhDpOjAKBggqhkjOPQQDAjBaMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRMwEQYDVQQDDAphY3Jhc2VydmVyMB4XDTE4MDMwNTE2MDgwMVoXDTI4 +MDMwMjE2MDgwMVowWjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKYWNy +YXNlcnZlcjB2MBAGByqGSM49AgEGBSuBBAAiA2IABB0cgtd3rItFJ4J84KSMhr/d +6q29DT8e2a6+cXIzZhcW/OYLUhltYETamwUqrvgYwKkGf1U2oscyhMmGeltHYFtf +aMCVxcAiyCa+lMJw2pgk9e1qljDsckrPinSVHptHiDAKBggqhkjOPQQDAgNnADBk +AjAkDSYpKd7jLKjaJGbu+/mCBu8I2fKQD/zcVxirtdMBD1Xhghs1hpLiGCpVdDDU +QewCMAjm9O81tKpDTWqeRW93kGRw+05g5XZe68/IRfCo3R9A4+xj1AqzY41/v9Dw +dFPjbg== +-----END CERTIFICATE----- diff --git a/docker/ssl/postgresql/postgresql.key b/docker/ssl/postgresql/postgresql.key new file mode 100644 index 000000000..29f877f6e --- /dev/null +++ b/docker/ssl/postgresql/postgresql.key @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAyKqW1QeWZET8794xQa4JINQJXTOUIt29jKhVnFf/MFBFD6s8y7nS5 +MP8PLzejdrugBwYFK4EEACKhZANiAAQdHILXd6yLRSeCfOCkjIa/3eqtvQ0/Htmu +vnFyM2YXFvzmC1IZbWBE2psFKq74GMCpBn9VNqLHMoTJhnpbR2BbX2jAlcXAIsgm +vpTCcNqYJPXtapYw7HJKz4p0lR6bR4g= +-----END EC PRIVATE KEY----- diff --git a/examples/golang/src/example/example.go b/examples/golang/src/example/example.go index aced23864..a6eead5a0 100644 --- a/examples/golang/src/example/example.go +++ b/examples/golang/src/example/example.go @@ -1,3 +1,5 @@ +// +build go1.8 + // Copyright 2016, Cossack Labs Limited // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/examples/golang/src/example_with_zone/example_with_zone.go b/examples/golang/src/example_with_zone/example_with_zone.go index bb50b3b2d..24cba621c 100644 --- a/examples/golang/src/example_with_zone/example_with_zone.go +++ b/examples/golang/src/example_with_zone/example_with_zone.go @@ -1,3 +1,5 @@ +// +build go1.8 + // Copyright 2016, Cossack Labs Limited // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/firewall/firewall_implementation.go b/firewall/firewall_implementation.go deleted file mode 100644 index 00b9fc5e4..000000000 --- a/firewall/firewall_implementation.go +++ /dev/null @@ -1,29 +0,0 @@ -package firewall - -type Firewall struct { - - handlers []QueryHandlerInterface -} - -func (firewall *Firewall) AddHandler(handler QueryHandlerInterface){ - - firewall.handlers = append(firewall.handlers, handler) -} - -func (firewall *Firewall) RemoveHandler(handler QueryHandlerInterface){ - for index, handlerFromRange := range firewall.handlers{ - if handlerFromRange == handler { - firewall.handlers = append(firewall.handlers[:index], firewall.handlers[index+1:]...) - } - } -} - -func (firewall *Firewall) HandleQuery(query string) error{ - - for _, handler := range firewall.handlers{ - if err := handler.CheckQuery(query); err != nil{ - return err - } - } - return nil -} diff --git a/firewall/firewall_test.go b/firewall/firewall_test.go deleted file mode 100644 index 3dc4bb0e3..000000000 --- a/firewall/firewall_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package firewall - -import ( - "testing" - "github.com/cossacklabs/acra/firewall/handlers" -) - - -func TestWhitelistFirewall(t *testing.T) { - - sqlSelectQueries := []string { - "SELECT * FROM Schema.Tables;", - "SELECT Student_ID FROM STUDENT;", - "SELECT * FROM STUDENT;", - "SELECT * FROM STUDENT;", - "SELECT * FROM STUDENT;", - "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", - "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", - "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", - "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", - "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", - "SELECT SUM(Salary)FROM Employee WHERE Emp_Age < 30;", - "SELECT AVG(Price)FROM Products;", - //"SELECT * FROM Schema.views;", - } - - sqlInsertQueries := []string { - //"INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');", - "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", - "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", - "INSERT INTO SalesStaff3 (StaffID, FullName)", - "INSERT INTO SalesStaff3 (StaffID, FullName)", - "INSERT INTO SalesStaff3 (StaffID, FullName)", - "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", - "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", - "INSERT INTO Production.UnitMeasure (Name, UnitMeasureCode, ModifiedDate) VALUES (N'Square Yards', N'Y2', GETDATE());", - "INSERT INTO T1 DEFAULT VALUES;", - "INSERT INTO dbo.Points (PointValue) VALUES (CONVERT(Point, '1,5'));", - "INSERT INTO dbo.Points (PointValue) VALUES (CAST ('1,99' AS Point));", - } - - whitelistHandler, err := handlers.NewWhitelistHandler([]string{"SELECT * FROM Schema.Tables;", "SELECT Student_ID FROM STUDENT;", "SELECT * FROM STUDENT;"}) - if err != nil { - t.Fatal("can't create whitelist handler") - } - whitelistHandler.AddQueriesToWhitelist(sqlSelectQueries) - whitelistHandler.AddQueriesToWhitelist(sqlInsertQueries) - - firewall := &Firewall{} - - //set our firewall to use whitelist for query evaluating - firewall.AddHandler(whitelistHandler) - - for _, query := range sqlSelectQueries{ - err = firewall.HandleQuery(query) - if err != nil { - t.Fatal(err) - } - } - - for _, query := range sqlInsertQueries{ - err = firewall.HandleQuery(query) - if err != nil { - t.Fatal(err) - } - } - - //firewall should block this query because it is not in whitelist - err = firewall.HandleQuery("SELECT * FROM Schema.views;") - if err == nil { - t.Fatal(err) - } - - //ditto - err = firewall.HandleQuery("INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');") - if err == nil { - t.Fatal(err) - } -} - -func TestBlacklistFirewall(t *testing.T) { - sqlSelectQueries := []string { - "SELECT * FROM Schema.Tables;", - "SELECT * FROM Schema.Tables;", - "SELECT * FROM Schema.Tables;", - "SELECT Student_ID FROM STUDENT;", - "SELECT * FROM STUDENT;", - "SELECT EMP_ID, NAME FROM EMPLOYEE_TBL WHERE EMP_ID = '0000';", - "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;", - "SELECT EMP_ID, LAST_NAME FROM EMPLOYEE_TBL WHERE CITY = 'INDIANAPOLIS' ORDER BY EMP_ID asc;", - "SELECT Name, Age FROM Patients WHERE Age > 40 GROUP BY Age ORDER BY Name;", - "SELECT COUNT(CustomerID), Country FROM Customers GROUP BY Country;", - "SELECT SUM(Salary) FROM Employee WHERE Emp_Age < 30;", - //"SELECT AVG(Price)FROM Products;", - "SELECT * FROM Schema.views;", - } - - sqlInsertQueries := []string { - //"INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');", - "INSERT SalesStaff1 VALUES (2, 'Michael', 'Blythe'), (3, 'Linda', 'Mitchell'),(4, 'Jillian', 'Carson'), (5, 'Garrett', 'Vargas');", - "INSERT INTO SalesStaff2 (StaffGUID, FirstName, LastName) VALUES (NEWID(), 'Stephen', 'Jiang');", - "INSERT INTO SalesStaff3 (StaffID, FullName)", - "INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway');", - "INSERT INTO Customers (CustomerName, City, Country) VALUES ('Cardinal', 'Stavanger', 'Norway');", - "INSERT INTO Production.UnitMeasure (Name, UnitMeasureCode, ModifiedDate) VALUES (N'Square Yards', N'Y2', GETDATE());", - "INSERT INTO T1 DEFAULT VALUES;", - "INSERT INTO dbo.Points (PointValue) VALUES (CONVERT(Point, '1,5'));", - "INSERT INTO dbo.Points (PointValue) VALUES (CAST ('1,99' AS Point));", - } - - blackList := [] string { - "INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');", - "SELECT AVG(Price) FROM Products;", - } - - blacklistHandler, err := handlers.NewBlacklistHandler(blackList) - if err != nil { - t.Fatal("can't create blacklist handler") - } - - firewall := &Firewall{} - - //set our firewall to use blacklist for query evaluating - firewall.AddHandler(blacklistHandler) - - for _, query := range sqlSelectQueries{ - err = firewall.HandleQuery(query) - if err != nil { - t.Fatal(err) - } - } - - for _, query := range sqlInsertQueries{ - err = firewall.HandleQuery(query) - if err != nil { - t.Fatal(err) - } - } - - testQuery := "INSERT INTO dbo.Points (PointValue) VALUES (CONVERT(Point, '1,5'));"; - - blacklistHandler.AddQueriesToBlacklist([]string{testQuery}) - - err = firewall.HandleQuery(testQuery) - //firewall should block this query by throwing error - if err == nil { - t.Fatal(err) - } - - firewall.RemoveHandler(blacklistHandler) - - err = firewall.HandleQuery(testQuery) - //firewall should not block this query because we removed blacklist handler, err should be nil - if err != nil { - t.Fatal(err) - } - - //again set our firewall to use blacklist for query evaluating - firewall.AddHandler(blacklistHandler) - - //now firewall should block testQuery by throwing error - if err != nil { - t.Fatal(err) - } - - blacklistHandler.RemoveQueriesFromBlacklist([]string{testQuery}) - - err = firewall.HandleQuery(testQuery) - //now firewall should not block testQuery - if err != nil { - t.Fatal(err) - } - -} \ No newline at end of file diff --git a/firewall/handlers/blacklist_handler.go b/firewall/handlers/blacklist_handler.go deleted file mode 100644 index 3147fe61a..000000000 --- a/firewall/handlers/blacklist_handler.go +++ /dev/null @@ -1,45 +0,0 @@ -package handlers - -import ( - "errors" -) - -type BlacklistHandler struct { - blackQueries[] string -} - -var ErrQueryInBlacklist = errors.New("query in blacklist") - -func NewBlacklistHandler(blackQueries []string) (*BlacklistHandler, error) { - - uniqueBlackQueries := removeDuplicates(blackQueries) - return &BlacklistHandler{blackQueries:uniqueBlackQueries}, nil -} - -func(handler * BlacklistHandler) CheckQuery(query string) error { - - yes, _ := contains(handler.blackQueries, query) - if yes { - return ErrQueryInBlacklist - } - return nil -} - -func(handler * BlacklistHandler) AddQueriesToBlacklist(queries []string) { - - for _, query := range queries{ - handler.blackQueries = append(handler.blackQueries, query) - } - - handler.blackQueries = removeDuplicates(handler.blackQueries) -} - -func(handler * BlacklistHandler) RemoveQueriesFromBlacklist(queries []string){ - - for _, query := range queries{ - yes, index := contains(handler.blackQueries, query) - if yes { - handler.blackQueries = append(handler.blackQueries[:index], handler.blackQueries[index+1:]...) - } - } -} \ No newline at end of file diff --git a/firewall/handlers/handlers_test.go b/firewall/handlers/handlers_test.go deleted file mode 100644 index 5467f11f9..000000000 --- a/firewall/handlers/handlers_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package handlers - -import ( - "testing" -) - -func TestUtilities(t *testing.T){ - - //Test 1 - expected := []string {"x", "y", "z"} - - input := []string {"x", "y", "z", "x", "y"} - - output := removeDuplicates(input) - - if !areEqual(output, expected) { - t.Fatal("unexpected result") - } - - //Test 2 - expected = []string {"@lagovas", "@vixentael", "@secumod"} - - input = []string {"@lagovas", "@vixentael", "@secumod", "@lagovas", "@vixentael", "@secumod", "@lagovas", "@vixentael", "@secumod"} - - output = removeDuplicates(input) - - if !areEqual(output, expected) { - t.Fatal("unexpected result") - } - -} - -func areEqual(a []string, b []string) bool { - if len(a) != len(b){ - return false - } - - for index := 0; index < len(a); index++{ - if a[index] != b[index]{ - return false - } - } - - return true -} diff --git a/firewall/handlers/handlers_util.go b/firewall/handlers/handlers_util.go deleted file mode 100644 index d6ebaf04a..000000000 --- a/firewall/handlers/handlers_util.go +++ /dev/null @@ -1,26 +0,0 @@ -package handlers - -func removeDuplicates(input []string) []string { - - keys := make(map[string] bool) - var result []string - for _, entry := range input{ - if _, value := keys[entry]; !value { - keys[entry] = true - result = append(result, entry) - } - } - return result - -} - -func contains(queries []string, query string) (bool, int) { - - for index, queryFromRange := range queries { - if queryFromRange == query { - - return true, index - } - } - return false, 0 -} \ No newline at end of file diff --git a/firewall/handlers/whitelist_handler.go b/firewall/handlers/whitelist_handler.go deleted file mode 100644 index 826270441..000000000 --- a/firewall/handlers/whitelist_handler.go +++ /dev/null @@ -1,44 +0,0 @@ -package handlers - -import ( - "errors" -) - -type WhitelistHandler struct { - whiteQueries[] string -} - -var ErrQueryNotInWhitelist = errors.New("query not in whitelist") - -func NewWhitelistHandler(whiteQueries []string) (*WhitelistHandler, error) { - - uniqueWhiteQueries := removeDuplicates(whiteQueries) - return &WhitelistHandler{whiteQueries:uniqueWhiteQueries}, nil -} - -func(handler * WhitelistHandler) CheckQuery(query string) error { - - yes, _ := contains(handler.whiteQueries, query) - if !yes { - return ErrQueryNotInWhitelist - } - return nil -} - -func(handler * WhitelistHandler) AddQueriesToWhitelist(queries []string) { - - for _, query := range queries { - handler.whiteQueries = append(handler.whiteQueries, query) - } - handler.whiteQueries = removeDuplicates(handler.whiteQueries) -} - -func (handler * WhitelistHandler) RemoveQueriesFromWhitelist(queries []string) { - - for _, query := range handler.whiteQueries { - yes, index := contains(handler.whiteQueries, query) - if yes { - handler.whiteQueries = append(handler.whiteQueries[:index], handler.whiteQueries[index+1:]...) - } - } -} \ No newline at end of file diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 96254207b..f0f7bee75 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -1,3 +1,5 @@ +// +build go1.8 + // Copyright 2016, Cossack Labs Limited // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/keystore/filesystem_store.go b/keystore/filenames.go similarity index 97% rename from keystore/filesystem_store.go rename to keystore/filenames.go index 063468f9d..1dbe1c70a 100644 --- a/keystore/filesystem_store.go +++ b/keystore/filenames.go @@ -23,6 +23,7 @@ var lock = sync.RWMutex{} const ( POISON_KEY_FILENAME = ".poison_key/poison_key" + BASIC_AUTH_KEY_FILENAME = "auth_key" ) func getZoneKeyFilename(id []byte) string { diff --git a/keystore/keystore.go b/keystore/keystore.go index 80a524664..60179fe65 100644 --- a/keystore/keystore.go +++ b/keystore/keystore.go @@ -14,19 +14,45 @@ package keystore import ( + "crypto/rand" + "encoding/base64" "errors" - "github.com/cossacklabs/themis/gothemis/keys" + "fmt" + "os" "strings" + + "github.com/cossacklabs/themis/gothemis/cell" + "github.com/cossacklabs/themis/gothemis/keys" ) const ( - DEFAULT_KEY_DIR_SHORT = ".acrakeys" - VALID_CHARS = "_- " - MAX_CLIENT_ID_LENGTH = 256 - MIN_CLIENT_ID_LENGTH = 5 + DEFAULT_KEY_DIR_SHORT = ".acrakeys" + VALID_CHARS = "_- " + MAX_CLIENT_ID_LENGTH = 256 + MIN_CLIENT_ID_LENGTH = 5 + BASIC_AUTH_KEY_LENGTH = 32 + ACRA_MASTER_KEY_VAR_NAME = "ACRA_MASTER_KEY" + // SYMMETRIC_KEY_LENGTH in bytes for master key + SYMMETRIC_KEY_LENGTH = 32 ) -var ErrInvalidClientId = errors.New("Invalid client id") +var ErrInvalidClientId = errors.New("invalid client id") +var ErrEmptyMasterKey = errors.New("master key is empty") +var ErrMasterKeyIncorrectLength = fmt.Errorf("master key must have %v length in bytes", SYMMETRIC_KEY_LENGTH) + +// GenerateSymmetricKey return new generated symmetric key that must used in keystore as master key and will comply +// our requirements +func GenerateSymmetricKey() ([]byte, error) { + key := make([]byte, SYMMETRIC_KEY_LENGTH) + n, err := rand.Read(key) + if err != nil { + return nil, err + } + if n != SYMMETRIC_KEY_LENGTH { + return nil, ErrMasterKeyIncorrectLength + } + return key, nil +} func ValidateId(client_id []byte) bool { if len(client_id) < MIN_CLIENT_ID_LENGTH || len(client_id) > MAX_CLIENT_ID_LENGTH { @@ -41,6 +67,54 @@ func ValidateId(client_id []byte) bool { return true } +// ValidateMasterKey do validation of symmetric master key and return nil if pass check +func ValidateMasterKey(key []byte) error { + if len(key) != SYMMETRIC_KEY_LENGTH { + return ErrMasterKeyIncorrectLength + } + return nil +} + +// GetMasterKeyFromEnvironment return master key from environment variable with name ACRA_MASTER_KEY_VAR_NAME +func GetMasterKeyFromEnvironment() (key []byte, err error) { + b64value := os.Getenv(ACRA_MASTER_KEY_VAR_NAME) + if len(b64value) == 0 { + return nil, ErrEmptyMasterKey + } + key, err = base64.StdEncoding.DecodeString(b64value) + if err != nil { + return + } + if err = ValidateMasterKey(key); err != nil { + return + } + return +} + +type KeyEncryptor interface { + Encrypt(key, context []byte) ([]byte, error) + Decrypt(key, context []byte) ([]byte, error) +} + +type SCellKeyEncryptor struct { + scell *cell.SecureCell +} + +func NewSCellKeyEncryptor(masterKey []byte) (*SCellKeyEncryptor, error) { + return &SCellKeyEncryptor{scell: cell.New(masterKey, cell.CELL_MODE_SEAL)}, nil +} + +// EncryptKey return encrypted key using masterKey and context +func (encryptor *SCellKeyEncryptor) Encrypt(key, context []byte) ([]byte, error) { + encrypted, _, err := encryptor.scell.Protect(key, context) + return encrypted, err +} + +// DecryptKey return decrypted key using masterKey and context +func (encryptor *SCellKeyEncryptor) Decrypt(key, context []byte) ([]byte, error) { + return encryptor.scell.Unprotect(key, nil, context) +} + type SecureSessionKeyStore interface { GetPrivateKey(id []byte) (*keys.PrivateKey, error) GetPeerPublicKey(id []byte) (*keys.PublicKey, error) @@ -58,8 +132,8 @@ type KeyStore interface { GenerateServerKeys(id []byte) error // generate key pair for data encryption/decryption GenerateDataEncryptionKeys(id []byte) error - GetPoisonKeyPair() (*keys.Keypair, error) + GetAuthKey(remove bool) ([]byte, error) Reset() } diff --git a/keystore/keystore_test.go b/keystore/keystore_test.go index 3d7e59861..a232d9c56 100644 --- a/keystore/keystore_test.go +++ b/keystore/keystore_test.go @@ -15,6 +15,8 @@ package keystore import ( "bytes" + "encoding/base64" + "os" "testing" ) @@ -76,3 +78,36 @@ func TestValidateId(t *testing.T) { t.Errorf("Incorrect false validation. <%s> took", max_id) } } + +func TestGetMasterKeyFromEnvironment(t *testing.T) { + if err := os.Setenv(ACRA_MASTER_KEY_VAR_NAME, ""); err != nil { + t.Fatal(err) + } + if _, err := GetMasterKeyFromEnvironment(); err != ErrEmptyMasterKey { + t.Fatal("expected ErrEmptyMasterKey") + } + key := []byte("some key") + if err := os.Setenv(ACRA_MASTER_KEY_VAR_NAME, base64.StdEncoding.EncodeToString(key)); err != nil { + t.Fatal(err) + } + + if _, err := GetMasterKeyFromEnvironment(); err != ErrMasterKeyIncorrectLength { + t.Fatal("expected ErrMasterKeyIncorrectLength error") + } + + key, err := GenerateSymmetricKey() + if err != nil { + t.Fatal(err) + } + if err := os.Setenv(ACRA_MASTER_KEY_VAR_NAME, base64.StdEncoding.EncodeToString(key)); err != nil { + t.Fatal(err) + } + + if envKey, err := GetMasterKeyFromEnvironment(); err != nil { + t.Fatal(err) + } else { + if !bytes.Equal(envKey, key) { + t.Fatal("keys not equal") + } + } +} diff --git a/keystore/proxy_keystore.go b/keystore/proxy_keystore.go index 999663d7f..d2d6a8d97 100644 --- a/keystore/proxy_keystore.go +++ b/keystore/proxy_keystore.go @@ -9,10 +9,11 @@ import ( type ProxyFileSystemKeyStore struct { directory string clientId []byte + encryptor KeyEncryptor } -func NewProxyFileSystemKeyStore(directory string, clientId []byte) (*ProxyFileSystemKeyStore, error) { - return &ProxyFileSystemKeyStore{directory: directory, clientId: clientId}, nil +func NewProxyFileSystemKeyStore(directory string, clientId []byte, encryptor KeyEncryptor) (*ProxyFileSystemKeyStore, error) { + return &ProxyFileSystemKeyStore{directory: directory, clientId: clientId, encryptor: encryptor}, nil } func (store *ProxyFileSystemKeyStore) GetPrivateKey(id []byte) (*keys.PrivateKey, error) { @@ -20,7 +21,11 @@ func (store *ProxyFileSystemKeyStore) GetPrivateKey(id []byte) (*keys.PrivateKey if err != nil { return nil, err } - return &keys.PrivateKey{Value: keyData}, nil + if privateKey, err := store.encryptor.Decrypt(keyData, id); err != nil { + return nil, err + } else { + return &keys.PrivateKey{Value: privateKey}, nil + } } func (store *ProxyFileSystemKeyStore) GetPeerPublicKey(id []byte) (*keys.PublicKey, error) { diff --git a/keystore/server_keystore.go b/keystore/server_keystore.go index ffdfa54de..22d118597 100644 --- a/keystore/server_keystore.go +++ b/keystore/server_keystore.go @@ -1,6 +1,7 @@ package keystore import ( + "crypto/rand" "errors" "fmt" "github.com/cossacklabs/acra/utils" @@ -11,15 +12,25 @@ import ( "os" "path/filepath" "runtime" + "sync" ) type FilesystemKeyStore struct { - keys map[string][]byte - directory string + keys map[string][]byte + privateKeyDirectory string + publicKeyDirectory string + directory string + lock *sync.RWMutex + encryptor KeyEncryptor } -func NewFilesystemKeyStore(directory string) (*FilesystemKeyStore, error) { - directory, err := utils.AbsPath(directory) +func NewFilesystemKeyStore(directory string, encryptor KeyEncryptor) (*FilesystemKeyStore, error) { + return NewFilesystemKeyStoreTwoPath(directory, directory, encryptor) +} + +func NewFilesystemKeyStoreTwoPath(privateKeyFolder, publicKeyFolder string, encryptor KeyEncryptor) (*FilesystemKeyStore, error) { + // check folder for private key + directory, err := utils.AbsPath(privateKeyFolder) if err != nil { return nil, err } @@ -28,30 +39,75 @@ func NewFilesystemKeyStore(directory string) (*FilesystemKeyStore, error) { log.Errorln(" key store folder has an incorrect permissions") return nil, errors.New("key store folder has an incorrect permissions") } - return &FilesystemKeyStore{directory: directory, keys: make(map[string][]byte)}, nil + if privateKeyFolder != publicKeyFolder { + // check folder for public key + directory, err = utils.AbsPath(privateKeyFolder) + if err != nil { + return nil, err + } + fi, err = os.Stat(directory) + if nil != err && !os.IsNotExist(err) { + return nil, err + } + } + return &FilesystemKeyStore{privateKeyDirectory: privateKeyFolder, publicKeyDirectory: publicKeyFolder, + keys: make(map[string][]byte), lock: &sync.RWMutex{}, encryptor: encryptor}, nil } -func (store *FilesystemKeyStore) generateKeyPair(filename string) (*keys.Keypair, error) { +func (store *FilesystemKeyStore) generateKeyPair(filename string, clientId []byte) (*keys.Keypair, error) { keypair, err := keys.New(keys.KEYTYPE_EC) if err != nil { return nil, err } - dirpath := filepath.Dir(store.getFilePath(filename)) - err = os.MkdirAll(dirpath, 0700) + privateKeysFolder := filepath.Dir(store.getPrivateKeyFilePath(filename)) + err = os.MkdirAll(privateKeysFolder, 0700) if err != nil { return nil, err } - err = ioutil.WriteFile(store.getFilePath(filename), keypair.Private.Value, 0600) + + publicKeysFolder := filepath.Dir(store.getPublicKeyFilePath(filename)) + err = os.MkdirAll(publicKeysFolder, 0700) if err != nil { return nil, err } - err = ioutil.WriteFile(store.getFilePath(fmt.Sprintf("%s.pub", filename)), keypair.Public.Value, 0644) + + encryptedPrivate, err := store.encryptor.Encrypt(keypair.Private.Value, clientId) + if err != nil { + return nil, err + } + err = ioutil.WriteFile(store.getPrivateKeyFilePath(filename), encryptedPrivate, 0600) + if err != nil { + return nil, err + } + err = ioutil.WriteFile(store.getPublicKeyFilePath(fmt.Sprintf("%s.pub", filename)), keypair.Public.Value, 0644) if err != nil { return nil, err } return keypair, nil } +func (store *FilesystemKeyStore) generateKey(filename string, length uint8) ([]byte, error) { + randomBytes := make([]byte, length) + _, err := rand.Read(randomBytes) + // Note that err == nil only if we read len(b) bytes. + if err != nil { + log.Error(err) + return nil, err + } + dirpath := filepath.Dir(store.getPrivateKeyFilePath(filename)) + err = os.MkdirAll(dirpath, 0700) + if err != nil { + log.Error(err) + return nil, err + } + err = ioutil.WriteFile(store.getPrivateKeyFilePath(filename), randomBytes, 0600) + if err != nil { + log.Error(err) + return nil, err + } + return randomBytes, nil +} + func (store *FilesystemKeyStore) GenerateZoneKey() ([]byte, []byte, error) { /* save private key in fs, return id and public key*/ var id []byte @@ -63,19 +119,23 @@ func (store *FilesystemKeyStore) GenerateZoneKey() ([]byte, []byte, error) { } } - keypair, err := store.generateKeyPair(getZoneKeyFilename(id)) + keypair, err := store.generateKeyPair(getZoneKeyFilename(id), id) if err != nil { return []byte{}, []byte{}, err } - lock.Lock() - defer lock.Unlock() + store.lock.Lock() + defer store.lock.Unlock() // cache key store.keys[getZoneKeyFilename(id)] = keypair.Private.Value return id, keypair.Public.Value, nil } -func (store *FilesystemKeyStore) getFilePath(filename string) string { - return fmt.Sprintf("%s%s%s", store.directory, string(os.PathSeparator), filename) +func (store *FilesystemKeyStore) getPrivateKeyFilePath(filename string) string { + return fmt.Sprintf("%s%s%s", store.privateKeyDirectory, string(os.PathSeparator), filename) +} + +func (store *FilesystemKeyStore) getPublicKeyFilePath(filename string) string { + return fmt.Sprintf("%s%s%s", store.publicKeyDirectory, string(os.PathSeparator), filename) } func (store *FilesystemKeyStore) GetZonePrivateKey(id []byte) (*keys.PrivateKey, error) { @@ -83,17 +143,20 @@ func (store *FilesystemKeyStore) GetZonePrivateKey(id []byte) (*keys.PrivateKey, return nil, ErrInvalidClientId } fname := getZoneKeyFilename(id) - lock.Lock() - defer lock.Unlock() + store.lock.Lock() + defer store.lock.Unlock() key, ok := store.keys[fname] if ok { log.Debugf("load cached key: %s", fname) return &keys.PrivateKey{Value: key}, nil } - privateKey, err := utils.LoadPrivateKey(store.getFilePath(fname)) + privateKey, err := utils.LoadPrivateKey(store.getPrivateKeyFilePath(fname)) if err != nil { return nil, err } + if privateKey.Value, err = store.encryptor.Decrypt(privateKey.Value, id); err != nil { + return nil, err + } log.Debugf("load key from fs: %s", fname) store.keys[fname] = privateKey.Value return privateKey, nil @@ -109,13 +172,13 @@ func (store *FilesystemKeyStore) HasZonePrivateKey(id []byte) bool { return false } fname := getZoneKeyFilename(id) - lock.RLock() - defer lock.RUnlock() + store.lock.RLock() + defer store.lock.RUnlock() _, ok := store.keys[fname] if ok { return true } - exists, _ := utils.FileExists(store.getFilePath(fname)) + exists, _ := utils.FileExists(store.getPrivateKeyFilePath(fname)) return exists } @@ -124,14 +187,14 @@ func (store *FilesystemKeyStore) GetPeerPublicKey(id []byte) (*keys.PublicKey, e return nil, ErrInvalidClientId } fname := getPublicKeyFilename(id) - lock.Lock() - defer lock.Unlock() + store.lock.Lock() + defer store.lock.Unlock() key, ok := store.keys[fname] if ok { log.Debugf("load cached key: %s", fname) return &keys.PublicKey{Value: key}, nil } - publicKey, err := utils.LoadPublicKey(store.getFilePath(fname)) + publicKey, err := utils.LoadPublicKey(store.getPublicKeyFilePath(fname)) if err != nil { return nil, err } @@ -152,10 +215,13 @@ func (store *FilesystemKeyStore) GetPrivateKey(id []byte) (*keys.PrivateKey, err log.Debugf("load cached key: %s", fname) return &keys.PrivateKey{Value: key}, nil } - privateKey, err := utils.LoadPrivateKey(store.getFilePath(fname)) + privateKey, err := utils.LoadPrivateKey(store.getPrivateKeyFilePath(fname)) if err != nil { return nil, err } + if privateKey.Value, err = store.encryptor.Decrypt(privateKey.Value, id); err != nil { + return nil, err + } log.Debugf("load key from fs: %s", fname) store.keys[fname] = privateKey.Value return privateKey, nil @@ -166,17 +232,20 @@ func (store *FilesystemKeyStore) GetServerDecryptionPrivateKey(id []byte) (*keys return nil, ErrInvalidClientId } fname := getServerDecryptionKeyFilename(id) - lock.Lock() - defer lock.Unlock() + store.lock.Lock() + defer store.lock.Unlock() key, ok := store.keys[fname] if ok { log.Debugf("load cached key: %s", fname) return &keys.PrivateKey{Value: key}, nil } - privateKey, err := utils.LoadPrivateKey(store.getFilePath(fname)) + privateKey, err := utils.LoadPrivateKey(store.getPrivateKeyFilePath(fname)) if err != nil { return nil, err } + if privateKey.Value, err = store.encryptor.Decrypt(privateKey.Value, id); err != nil { + return nil, err + } log.Debugf("load key from fs: %s", fname) store.keys[fname] = privateKey.Value return privateKey, nil @@ -187,7 +256,8 @@ func (store *FilesystemKeyStore) GenerateProxyKeys(id []byte) error { return ErrInvalidClientId } filename := getProxyKeyFilename(id) - _, err := store.generateKeyPair(filename) + + _, err := store.generateKeyPair(filename, id) if err != nil { return err } @@ -198,7 +268,7 @@ func (store *FilesystemKeyStore) GenerateServerKeys(id []byte) error { return ErrInvalidClientId } filename := getServerKeyFilename(id) - _, err := store.generateKeyPair(filename) + _, err := store.generateKeyPair(filename, id) if err != nil { return err } @@ -210,7 +280,7 @@ func (store *FilesystemKeyStore) GenerateDataEncryptionKeys(id []byte) error { if !ValidateId(id) { return ErrInvalidClientId } - _, err := store.generateKeyPair(getServerDecryptionKeyFilename(id)) + _, err := store.generateKeyPair(getServerDecryptionKeyFilename(id), id) if err != nil { return err } @@ -223,8 +293,8 @@ func (store *FilesystemKeyStore) Reset() { } func (store *FilesystemKeyStore) GetPoisonKeyPair() (*keys.Keypair, error) { - privatePath := store.getFilePath(POISON_KEY_FILENAME) - publicPath := store.getFilePath(fmt.Sprintf("%s.pub", POISON_KEY_FILENAME)) + privatePath := store.getPrivateKeyFilePath(POISON_KEY_FILENAME) + publicPath := store.getPublicKeyFilePath(fmt.Sprintf("%s.pub", POISON_KEY_FILENAME)) privateExists, err := utils.FileExists(privatePath) if err != nil { return nil, err @@ -238,6 +308,9 @@ func (store *FilesystemKeyStore) GetPoisonKeyPair() (*keys.Keypair, error) { if err != nil { return nil, err } + if private.Value, err = store.encryptor.Decrypt(private.Value, []byte(POISON_KEY_FILENAME)); err != nil { + return nil, err + } public, err := utils.LoadPublicKey(publicPath) if err != nil { return nil, err @@ -245,5 +318,24 @@ func (store *FilesystemKeyStore) GetPoisonKeyPair() (*keys.Keypair, error) { return &keys.Keypair{Public: public, Private: private}, nil } log.Infoln("Generate poison key pair") - return store.generateKeyPair(POISON_KEY_FILENAME) + return store.generateKeyPair(POISON_KEY_FILENAME, []byte(POISON_KEY_FILENAME)) +} + +func (store *FilesystemKeyStore) GetAuthKey(remove bool) ([]byte, error) { + keyPath := store.getPrivateKeyFilePath(BASIC_AUTH_KEY_FILENAME) + keyExists, err := utils.FileExists(keyPath) + if err != nil { + log.Error(err) + return nil, err + } + if keyExists && !remove { + key, err := utils.ReadFile(keyPath) + if err != nil { + log.Error(err) + return nil, err + } + return key, nil + } + log.Infof("Generate basic auth key for AcraConfigUI to %v", keyPath) + return store.generateKey(BASIC_AUTH_KEY_FILENAME, BASIC_AUTH_KEY_LENGTH) } diff --git a/keystore/filesystem_store_test.go b/keystore/server_keystore_test.go similarity index 63% rename from keystore/filesystem_store_test.go rename to keystore/server_keystore_test.go index 0fc7d3fec..1de5ae707 100644 --- a/keystore/filesystem_store_test.go +++ b/keystore/server_keystore_test.go @@ -14,13 +14,39 @@ package keystore import ( + "bytes" "fmt" "github.com/cossacklabs/acra/utils" + "io/ioutil" "os" "path/filepath" "testing" ) +func testGenerateKeyPair(store *FilesystemKeyStore, t *testing.T) { + clientId := []byte("some test id") + file, err := ioutil.TempFile("", "test_generate_key_pair") + if err != nil { + t.Fatal(err) + } + // create temp file with random name to use it as not-existed path + path := file.Name() + file.Close() + defer os.Remove(path) + keypair, err := store.generateKeyPair(path, clientId) + if err != nil { + t.Fatal(err) + } + encryptedKey, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + // check that returned key != stored on filesystem data + if bytes.Equal(encryptedKey, keypair.Private.Value) { + t.Fatal("keys are equal") + } +} + func testGeneral(store *FilesystemKeyStore, t *testing.T) { if store.HasZonePrivateKey([]byte("non-existent key")) { t.Fatal("Expected false on non-existent key") @@ -55,7 +81,7 @@ func testGeneratingDataEncryptionKeys(store *FilesystemKeyStore, t *testing.T) { t.Fatal(err) } exists, err := utils.FileExists( - store.getFilePath( + store.getPrivateKeyFilePath( getServerDecryptionKeyFilename(testId))) if err != nil { t.Fatal(err) @@ -65,7 +91,7 @@ func testGeneratingDataEncryptionKeys(store *FilesystemKeyStore, t *testing.T) { } exists, err = utils.FileExists( - fmt.Sprintf("%s.pub", store.getFilePath( + fmt.Sprintf("%s.pub", store.getPublicKeyFilePath( getServerDecryptionKeyFilename(testId)))) if err != nil { t.Fatal(err) @@ -86,7 +112,7 @@ func testGenerateServerKeys(store *FilesystemKeyStore, t *testing.T) { fmt.Sprintf("%s.pub", getServerKeyFilename(testId)), } for _, name := range expectedPaths { - absPath := store.getFilePath(name) + absPath := store.getPrivateKeyFilePath(name) exists, err := utils.FileExists(absPath) if err != nil { t.Fatal(err) @@ -108,7 +134,7 @@ func testGenerateProxyKeys(store *FilesystemKeyStore, t *testing.T) { fmt.Sprintf("%s.pub", getProxyKeyFilename(testId)), } for _, name := range expectedPaths { - absPath := store.getFilePath(name) + absPath := store.getPrivateKeyFilePath(name) exists, err := utils.FileExists(absPath) if err != nil { t.Fatal(err) @@ -128,10 +154,10 @@ func testReset(store *FilesystemKeyStore, t *testing.T) { t.Fatal(err) } store.Reset() - if err := os.Remove(store.getFilePath(getServerKeyFilename(testId))); err != nil { + if err := os.Remove(store.getPrivateKeyFilePath(getServerKeyFilename(testId))); err != nil { t.Fatal(err) } - if err := os.Remove(fmt.Sprintf("%s.pub", store.getFilePath(getServerKeyFilename(testId)))); err != nil { + if err := os.Remove(fmt.Sprintf("%s.pub", store.getPublicKeyFilePath(getServerKeyFilename(testId)))); err != nil { t.Fatal(err) } @@ -141,18 +167,34 @@ func testReset(store *FilesystemKeyStore, t *testing.T) { } func TestFilesystemKeyStore(t *testing.T) { - keyDirectory := fmt.Sprintf(".%s%s", string(filepath.Separator), "keys") - os.MkdirAll(keyDirectory, 0700) + privateKeyDirectory := fmt.Sprintf(".%s%s", string(filepath.Separator), "keys") + os.MkdirAll(privateKeyDirectory, 0700) defer func() { - os.RemoveAll(keyDirectory) + os.RemoveAll(privateKeyDirectory) }() - store, err := NewFilesystemKeyStore(keyDirectory) + + encryptor, err := NewSCellKeyEncryptor([]byte("some key")) + if err != nil { + t.Fatal(err) + } + publicKeyDirectory := fmt.Sprintf(".%s%s", string(filepath.Separator), "public_keys") + os.MkdirAll(publicKeyDirectory, 0700) + defer func() { + os.RemoveAll(publicKeyDirectory) + }() + generalStore, err := NewFilesystemKeyStore(privateKeyDirectory, encryptor) if err != nil { - t.Fatal("error") + t.Fatal(err) + } + splitKeysStore, err := NewFilesystemKeyStoreTwoPath(privateKeyDirectory, publicKeyDirectory, encryptor) + if err != nil { + t.Fatal(err) + } + for _, store := range []*FilesystemKeyStore{generalStore, splitKeysStore} { + testGeneral(store, t) + testGeneratingDataEncryptionKeys(store, t) + testGenerateProxyKeys(store, t) + testGenerateServerKeys(store, t) + testReset(store, t) } - testGeneral(store, t) - testGeneratingDataEncryptionKeys(store, t) - testGenerateProxyKeys(store, t) - testGenerateServerKeys(store, t) - testReset(store, t) } diff --git a/logging/cef_formatter.go b/logging/cef_formatter.go new file mode 100644 index 000000000..252a2ea34 --- /dev/null +++ b/logging/cef_formatter.go @@ -0,0 +1,218 @@ +package logging + +import ( + "bytes" + "fmt" + "github.com/sirupsen/logrus" + "os" + "strings" + "sync" + "time" +) + +// Almost compatible with CEF doc +// https://kc.mcafee.com/resources/sites/MCAFEE/content/live/CORP_KNOWLEDGEBASE/78000/KB78712/en_US/CEF_White_Paper_20100722.pdf +// +// Current implementation allows using any extension keys +// + +const defaultTimestampFormat = time.RFC3339 +const defaultCEFLogStart = "CEF:0" +const defaultHostName = "host" +const defaultMessageDivider = "|" + +// Default key names for the default fields +const ( + FieldKeyUnixTime = "unixTime" + FieldKeyProduct = "product" + FieldKeyVersion = "version" + FieldKeySeverity = "severity" + FieldKeyVendor = "vendor" + FieldKeyEventCode = "code" +) + +// CEFTextFormatter formats logs into text +type CEFTextFormatter struct { + // TimestampFormat to use for display when a full timestamp is printed + TimestampFormat string + + // QuoteEmptyFields will wrap empty fields in quotes if true + QuoteEmptyFields bool + + // By default 'CEF:0' + CEFPrefixString string + + // By default 'os.Hostname()' + HostName string + + // start log with syslog prefix automatically + ShouldAddSyslogPrefix bool + + sync.Once +} + +func (f *CEFTextFormatter) init(entry *logrus.Entry) { + f.ShouldAddSyslogPrefix = false + f.QuoteEmptyFields = true +} + +// Format renders a single log entry +func (f *CEFTextFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var b *bytes.Buffer + + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + + f.Do(func() { f.init(entry) }) + + timestampFormat := f.TimestampFormat + if timestampFormat == "" { + timestampFormat = defaultTimestampFormat + } + + hostname := f.HostName + if hostname == "" { + realHostName, err := os.Hostname() + if err != nil { + hostname = defaultHostName + } else { + hostname = realHostName + } + } + + logPrefix := f.CEFPrefixString + if logPrefix == "" { + logPrefix = defaultCEFLogStart + } + + // syslog prefix + // timestamp host + if f.ShouldAddSyslogPrefix { + b.WriteString(entry.Time.Format(timestampFormat)) + b.WriteByte(' ') + b.WriteString(hostname) + b.WriteByte(' ') + } + + // CEF:0 + b.WriteString(defaultCEFLogStart) + + // |Device Vendor|Device Product|Device Version|Signature ID|Name|Severity| + f.appendCEFLogPiece(b, entry.Data[FieldKeyVendor]) + f.appendCEFLogPiece(b, entry.Data[FieldKeyProduct]) + f.appendCEFLogPiece(b, entry.Data[FieldKeyVersion]) + f.appendCEFLogPiece(b, entry.Data[FieldKeyEventCode]) + + f.appendCEFLogPiece(b, entry.Message) + f.appendCEFLogPiece(b, severityByLevel(entry.Level)) + + b.WriteString(defaultMessageDivider) + + // Extension + + // actually, these fields should have only designated names according to the Extension Dictionary of CEF + extensionKeys := otherExtensionKeys(entry.Data) + for _, key := range extensionKeys { + f.appendKeyValue(b, key, entry.Data[key]) + } + + b.WriteByte('\n') + return b.Bytes(), nil +} + +func otherExtensionKeys(data logrus.Fields) []string { + extensionKeys := make([]string, 0, len(data)) + for k := range data { + + if k != FieldKeyVendor && k != FieldKeyProduct && k != FieldKeyVersion && + k != FieldKeyEventCode && k != FieldKeySeverity { + + extensionKeys = append(extensionKeys, k) + } + } + return extensionKeys +} + +func (f *CEFTextFormatter) appendCEFLogPiece(b *bytes.Buffer, value interface{}) { + b.WriteString(defaultMessageDivider) + f.appendValue(b, value) +} + +func (f *CEFTextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) { + preparedKey := prepareString(key) + + if f.needsQuoting(preparedKey) { + preparedKey = fmt.Sprintf("%q", preparedKey) + } + + b.WriteString(preparedKey) + b.WriteByte('=') + f.appendValue(b, value) + b.WriteByte(' ') +} + +func (f *CEFTextFormatter) appendValue(b *bytes.Buffer, value interface{}) { + stringVal, ok := value.(string) + if !ok { + stringVal = fmt.Sprint(value) + } + + stringVal = prepareString(stringVal) + + // CEF doesn't define using quotes + if len(stringVal) == 0 { + b.WriteString(" ") + } else { + b.WriteString(stringVal) + } +} + +func prepareString(value string) string { + stringVal := fmt.Sprint(value) + + // is it a valid way to remove any \t\n even inside line? + stringVal = strings.TrimSpace(stringVal) + stringVal = strings.Replace(stringVal, "\n", " ", -1) + stringVal = strings.Replace(stringVal, "\t", " ", -1) + stringVal = strings.Replace(stringVal, `\`, `\\`, -1) + stringVal = strings.Replace(stringVal, "|", `\|`, -1) + stringVal = strings.Replace(stringVal, `=`, `\=`, -1) + return stringVal +} + +func severityByLevel(level logrus.Level) int { + switch level { + case logrus.DebugLevel: + return 0 + case logrus.InfoLevel: + return 1 + case logrus.WarnLevel: + return 3 + case logrus.ErrorLevel: + return 6 + case logrus.FatalLevel: + return 8 + case logrus.PanicLevel: + return 10 + } + + return 0 +} + +func (f *CEFTextFormatter) needsQuoting(text string) bool { + if f.QuoteEmptyFields && len(text) == 0 { + return true + } + for _, ch := range text { + if !((ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch == '-' || ch == '.' || ch == '_' || ch == '/' || ch == '@' || ch == '^' || ch == '+') { + return true + } + } + return false +} diff --git a/logging/cef_formatter_test.go b/logging/cef_formatter_test.go new file mode 100644 index 000000000..e8b40e971 --- /dev/null +++ b/logging/cef_formatter_test.go @@ -0,0 +1,28 @@ +package logging + +import ( + "testing" +) + +func TestStringEscape(t *testing.T) { + testString := "small string" + modifiedString := prepareString(testString) + + if modifiedString != testString { + t.Errorf("Incorrect CEF string escaping <%s>", modifiedString) + } + + testString = "small | = string" + modifiedString = prepareString(testString) + + if modifiedString != "small \\| \\= string" { + t.Errorf("Incorrect CEF string escaping <%s>", modifiedString) + } + + testString = "small \t \n string" + modifiedString = prepareString(testString) + + if modifiedString != "small string" { + t.Errorf("Incorrect CEF string escaping <%s>", modifiedString) + } +} diff --git a/logging/event_codes.go b/logging/event_codes.go new file mode 100644 index 000000000..8b54c29c8 --- /dev/null +++ b/logging/event_codes.go @@ -0,0 +1,76 @@ +package logging + +const ( + // 100 .. 200 some events + EventCodeGeneral = 100 + + // 500 .. 600 errors + EventCodeErrorGeneral = 500 + EventCodeErrorWrongParam = 501 + + // processes + EventCodeErrorCantStartService = 505 + EventCodeErrorCantForkProcess = 506 + EventCodeErrorWrongConfiguration = 507 + EventCodeErrorCantReadServiceConfig = 508 + EventCodeErrorCantCloseConnectionToService = 509 + + // keys + EventCodeErrorCantInitKeyStore = 510 + EventCodeErrorCantReadKeys = 511 + + // system events + EventCodeErrorCantGetFileDescriptor = 520 + EventCodeErrorCantOpenFileByDescriptor = 521 + EventCodeErrorFileDescriptionIsNotValid = 522 + EventCodeErrorCantRegisterSignalHandler = 523 + + // transport / networks + EventCodeErrorCantStartListenConnections = 530 + EventCodeErrorCantStopListenConnections = 531 + EventCodeErrorTransportConfiguration = 532 + EventCodeErrorCantAcceptNewConnections = 533 + EventCodeErrorCantStartConnection = 534 + EventCodeErrorCantHandleSecureSession = 535 + EventCodeErrorCantCloseConnection = 536 + EventCodeErrorCantInitClientSession = 537 + EventCodeErrorCantWrapConnection = 538 + EventCodeErrorConnectionDroppedByTimeout = 539 + + // database + EventCodeErrorCantConnectToDB = 540 + EventCodeErrorCantCloseConnectionDB = 541 + + // config UI + EventCodeErrorCantReadTemplate = 550 + EventCodeErrorRequestMethodNotAllowed = 551 + EventCodeErrorCantParseRequestData = 552 + EventCodeErrorCantGetCurrentConfig = 553 + EventCodeErrorCantSetNewConfig = 554 + EventCodeErrorCantHashPassword = 555 + EventCodeErrorCantGetAuthData = 556 + EventCodeErrorCantParseAuthData = 557 + EventCodeErrorCantDumpConfig = 558 + + // acracensor + EventCodeErrorCensorQueryIsNotAllowed = 560 + EventCodeErrorCensorSetupError = 561 + + // response proxy + EventCodeErrorResponseProxyCantWriteToDB = 570 + EventCodeErrorResponseProxyCantReadFromClient = 571 + EventCodeErrorResponseProxyCantWriteToClient = 572 + EventCodeErrorResponseProxyCantReadFromServer = 573 + EventCodeErrorResponseProxyCantWriteToServer = 574 + EventCodeErrorResponseProxyCantProcessColumn = 575 + EventCodeErrorResponseProxyCantProcessRow = 576 + + // decryptor + EventCodeErrorCantInitDecryptor = 580 + EventCodeErrorDecryptorCantDecryptBinary = 581 + EventCodeErrorDecryptorCantSkipBeginInBlock = 582 + EventCodeErrorDecryptorCantHandleRecognizedPoisonRecord = 583 + EventCodeErrorDecryptorCantInitializeTLS = 584 + EventCodeErrorDecryptorCantSetDeadlineToClientConnection = 585 + EventCodeErrorDecryptorCantDecryptSymmetricKey = 586 +) diff --git a/logging/log_formatters.go b/logging/log_formatters.go new file mode 100644 index 000000000..12a0cbd98 --- /dev/null +++ b/logging/log_formatters.go @@ -0,0 +1,170 @@ +package logging + +import ( + "fmt" + "github.com/cossacklabs/acra/utils" + "github.com/sirupsen/logrus" + "sync" + "time" +) + +// ---------- custom Loggers +// inspired by "github.com/bshuster-repo/logrus-logstash-hook" +// ---------- + +// TextFormatter returns a default logrus.TextFormatter with specific settings +func TextFormatter() logrus.Formatter { + return &logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: time.RFC3339, + QuoteEmptyFields: true} +} + +// JSONFormatter returns a AcraJSONFormatter +func JSONFormatter(fields logrus.Fields) logrus.Formatter { + for k, v := range extraJSONFields { + if _, ok := fields[k]; !ok { + fields[k] = v + } + } + + return AcraJSONFormatter{ + Formatter: &logrus.JSONFormatter{ + FieldMap: JSONFieldMap, + TimestampFormat: time.RFC3339, + }, + Fields: fields, + lock: &sync.RWMutex{}, + } +} + +// CEFFormatter returns a AcraCEFFormatter +func CEFFormatter(fields logrus.Fields) logrus.Formatter { + for k, v := range extraJSONFields { + if _, ok := fields[k]; !ok { + fields[k] = v + } + } + + for k, v := range extraCEFFields { + if _, ok := fields[k]; !ok { + fields[k] = v + } + } + + return AcraCEFFormatter{ + CEFTextFormatter: CEFTextFormatter{ + TimestampFormat: time.RFC3339, + }, + Fields: fields, + lock: &sync.RWMutex{}, + } +} + +// --------------------------- + +// Using a pool to re-use of old entries when formatting messages. +// It is used in the Fire function. +var entryPool = sync.Pool{ + New: func() interface{} { + return &logrus.Entry{} + }, +} + +// copyEntry copies the entry `e` to a new entry and then adds all the fields in `fields` that are missing in the new entry data. +// It uses `entryPool` to re-use allocated entries. +func copyEntry(e *logrus.Entry, fields logrus.Fields) *logrus.Entry { + ne := entryPool.Get().(*logrus.Entry) + ne.Message = e.Message + ne.Level = e.Level + ne.Time = e.Time + ne.Data = logrus.Fields{} + for k, v := range fields { + ne.Data[k] = v + } + for k, v := range e.Data { + ne.Data[k] = v + } + return ne +} + +// releaseEntry puts the given entry back to `entryPool`. It must be called if copyEntry is called. +func releaseEntry(e *logrus.Entry) { + entryPool.Put(e) +} + +// AcraCustomFormatter represents a format with specific fields. +// It has logrus.Formatter which formats the entry and logrus.Fields which +// are added to the JSON/CEF message if not given in the entry data. +// +// Note: use the `JSONFormatter` function to set a default AcraJSON formatter. +type AcraJSONFormatter struct { + logrus.Formatter + logrus.Fields + lock *sync.RWMutex +} + +type AcraCEFFormatter struct { + CEFTextFormatter + logrus.Fields + lock *sync.RWMutex +} + +var ( + // to be re-defined + extraJSONFields = logrus.Fields{ + FieldKeyProduct: "acra", + FieldKeyUnixTime: 0, + FieldKeyVersion: utils.VERSION, + } + + // to be re-defined + extraCEFFields = logrus.Fields{ + FieldKeyVendor: "cossacklabs", + FieldKeyEventCode: EventCodeGeneral, + } + + JSONFieldMap = logrus.FieldMap{ + logrus.FieldKeyTime: "timestamp", + logrus.FieldKeyMsg: "msg", + logrus.FieldKeyLevel: "level", + } +) + +// Format formats an entry to a AcraJSON format according to the given Formatter and Fields. +// +// Note: the given entry is copied and not changed during the formatting process. +func (f AcraJSONFormatter) Format(e *logrus.Entry) ([]byte, error) { + // unix time in milliseconds + f.lock.Lock() + f.Fields[FieldKeyUnixTime] = unixTimeWithMilliseconds(e) + f.lock.Unlock() + + ne := copyEntry(e, f.Fields) + dataBytes, err := f.Formatter.Format(ne) + releaseEntry(ne) + return dataBytes, err +} + +// Format formats an entry to a AcraCEF format according to the given Formatter and Fields. +// +// Note: the given entry is copied and not changed during the formatting process. +func (f AcraCEFFormatter) Format(e *logrus.Entry) ([]byte, error) { + // unix time in milliseconds + f.lock.Lock() + f.Fields[FieldKeyUnixTime] = unixTimeWithMilliseconds(e) + f.lock.Unlock() + + ne := copyEntry(e, f.Fields) + dataBytes, err := f.CEFTextFormatter.Format(ne) + releaseEntry(ne) + return dataBytes, err +} + +func unixTimeWithMilliseconds(e *logrus.Entry) string { + nanos := e.Time.UnixNano() + millis := nanos / 1000000 + millisf := float64(millis) / 1000.0 + + return fmt.Sprintf("%.3f", millisf) +} diff --git a/logging/logging.go b/logging/logging.go new file mode 100644 index 000000000..e096fbce4 --- /dev/null +++ b/logging/logging.go @@ -0,0 +1,59 @@ +// Copyright 2016, Cossack Labs Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package logging + +import ( + "fmt" + log "github.com/sirupsen/logrus" + "os" + "strings" +) + +const ( + LOG_DEBUG = iota + LOG_VERBOSE + LOG_DISCARD +) + +func SetLogLevel(level int) { + if level == LOG_DEBUG { + log.SetLevel(log.DebugLevel) + } else if level == LOG_VERBOSE { + log.SetLevel(log.InfoLevel) + } else if level == LOG_DISCARD { + log.SetLevel(log.WarnLevel) + } else { + panic(fmt.Sprintf("Incorrect log level - %v", level)) + } +} + +func CustomizeLogging(loggingFormat string, serviceName string) { + log.SetOutput(os.Stderr) + log.SetFormatter(logFormatterFor(loggingFormat, serviceName)) + + log.Debugf("Changed logging format to %s", loggingFormat) +} + +func logFormatterFor(loggingFormat string, serviceName string) log.Formatter { + loggingFormat = strings.ToLower(loggingFormat) + + if loggingFormat == "json" { + return JSONFormatter(log.Fields{FieldKeyProduct: serviceName}) + + } else if loggingFormat == "cef" { + return CEFFormatter(log.Fields{FieldKeyProduct: serviceName}) + } + + return TextFormatter() +} diff --git a/network/connection_wrapper.go b/network/connection_wrapper.go index 98dbf3ebe..b4b85e6e0 100644 --- a/network/connection_wrapper.go +++ b/network/connection_wrapper.go @@ -4,6 +4,10 @@ import ( "net" ) +type ConnectionTimeoutWrapper interface { + net.Conn +} + type ConnectionWrapper interface { WrapClient(id []byte, conn net.Conn) (net.Conn, error) WrapServer(conn net.Conn) (net.Conn, []byte, error) // conn, ClientId, error diff --git a/network/connections.go b/network/connections.go new file mode 100644 index 000000000..30deeef50 --- /dev/null +++ b/network/connections.go @@ -0,0 +1,34 @@ +package network + +import ( + log "github.com/sirupsen/logrus" + "net" + "sync" +) + +type ConnectionManager struct { + *sync.WaitGroup + Counter int + connections []*net.Conn +} + +func NewConnectionManager() *ConnectionManager { + cm := &ConnectionManager{} + cm.WaitGroup = &sync.WaitGroup{} + return cm +} + +func (cm *ConnectionManager) Incr() { + cm.Counter += 1 + log.Debugf("ConnectionManager.Add") + cm.WaitGroup.Add(1) +} + +func (cm *ConnectionManager) Done() { + cm.Counter-- + cm.WaitGroup.Done() +} + +func (cm *ConnectionManager) AddConnection(conn *net.Conn) { + cm.connections = append(cm.connections, conn) +} diff --git a/network/secure_session_wrapper.go b/network/secure_session_wrapper.go index 8531d2d89..f9a3e51e0 100644 --- a/network/secure_session_wrapper.go +++ b/network/secure_session_wrapper.go @@ -18,6 +18,7 @@ import ( "net" "github.com/cossacklabs/acra/keystore" + "github.com/cossacklabs/acra/logging" "github.com/cossacklabs/acra/utils" "github.com/cossacklabs/themis/gothemis/keys" "github.com/cossacklabs/themis/gothemis/session" @@ -29,10 +30,11 @@ type SessionCallback struct { } func (callback *SessionCallback) GetPublicKeyForId(ss *session.SecureSession, id []byte) *keys.PublicKey { - log.Infof("load public key for id <%v>", string(id)) + log.Infof("Load public key for id %v", string(id)) key, err := callback.keystorage.GetPeerPublicKey(id) if err != nil { - log.WithError(err).Errorf("can't load public key for id <%v>", string(id)) + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantReadKeys). + Errorf("Can't load public key for id %v", string(id)) return nil } return key diff --git a/network/tls_wrapper.go b/network/tls_wrapper.go index b78ae8960..7638337d3 100644 --- a/network/tls_wrapper.go +++ b/network/tls_wrapper.go @@ -6,7 +6,9 @@ import ( "io/ioutil" "net" + "github.com/cossacklabs/acra/logging" log "github.com/sirupsen/logrus" + "errors" ) type TLSConnectionWrapper struct { @@ -36,17 +38,26 @@ func (wrapper *TLSConnectionWrapper) WrapServer(conn net.Conn) (net.Conn, []byte } func NewTLSConfig(serverName string, caPath, keyPath, crtPath string) (*tls.Config, error) { - roots := x509.NewCertPool() + var roots *x509.CertPool + var err error + + if roots, err = x509.SystemCertPool(); err != nil { + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral). + Errorln("Can't load system ca certificates") + } + if roots == nil { + roots = x509.NewCertPool() + } if caPath != "" { caPem, err := ioutil.ReadFile(caPath) if err != nil { - log.WithError(err).Errorln("can't read root CA certificate") + log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral).Errorln("Can't read root CA certificate") return nil, err } - log.Debugln("add CA root certificate") + log.Debugln("Adding CA root certificate") if ok := roots.AppendCertsFromPEM(caPem); !ok { - log.Errorln("can't add CA certificate") - return nil, err + log.Errorln("Can't add CA certificate from PEM") + return nil, errors.New("can't add CA certificate") } } cer, err := tls.LoadX509KeyPair(crtPath, keyPath) diff --git a/network/utils.go b/network/utils.go index a50cf86ad..3f7296408 100644 --- a/network/utils.go +++ b/network/utils.go @@ -4,6 +4,7 @@ import ( "fmt" "net" url_ "net/url" + "os" ) // Dial connectionString like protocol://path where protocol is any supported via net.Dial (tcp|unix) @@ -19,6 +20,11 @@ func Dial(connectionString string) (net.Conn, error) { } } +type ListenerWithFileDescriptor interface { + net.Listener + File() (f *os.File, err error) +} + func Listen(connectionString string) (net.Listener, error) { url, err := url_.Parse(connectionString) if err != nil { @@ -34,3 +40,11 @@ func Listen(connectionString string) (net.Listener, error) { func BuildConnectionString(protocol, host string, port int, path string) string { return fmt.Sprintf("%s://%s:%v/%s", protocol, host, port, path) } + +func ListenerFileDescriptor(socket net.Listener) (uintptr, error) { + file, err := socket.(ListenerWithFileDescriptor).File() + if err != nil { + return 0, err + } + return file.Fd(), nil +} diff --git a/tests/Readme.md b/tests/Readme.md index d29becc52..483f01d32 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -33,7 +33,7 @@ pip3 install -r tests/requirements.txt If you want to customise database settings, pass them as environment variables: ```console -TEST_TLS=off TEST_SSL_MODE=allow TEST_DB_HOST=127.0.0.1 TEST_DB_USER=postgres TEST_DB_USER_PASSWORD=postgres TEST_DB_NAME=acra TEST_DB_PORT=5432 python3 tests/test.py +TEST_TLS=off TEST_SSL_MODE=allow TEST_DB_HOST=127.0.0.1 TEST_DB_USER=postgres TEST_DB_USER_PASSWORD=postgres TEST_DB_NAME=postgres TEST_DB_PORT=5432 python3 tests/test.py ``` or just use default database settings diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 000000000..d57c11aa1 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' + +services: + mydb: + image: postgres:9.6 + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_PORT: 5432 + PGDATA: /pgdata + ports: ["5432:5432"] \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 32802235c..e51458e8f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,6 @@ pythemis psycopg2 -sqlalchemy \ No newline at end of file +sqlalchemy +PyMySQL==0.8.0 +semver==2.7.9 +requests diff --git a/tests/test.py b/tests/test.py index c3ffd602d..63b14f521 100644 --- a/tests/test.py +++ b/tests/test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # coding: utf-8 +import contextlib import socket import json import time @@ -21,13 +22,18 @@ import subprocess import traceback import unittest +import re import stat -import collections from base64 import b64decode, b64encode from tempfile import NamedTemporaryFile from urllib.request import urlopen +from urllib.parse import urlparse +import collections +import shutil import psycopg2 +import pymysql +import semver import sqlalchemy as sa from sqlalchemy.exc import DatabaseError from sqlalchemy.dialects.postgresql import BYTEA @@ -38,12 +44,14 @@ from acrawriter import create_acrastruct - +DATA_MAX_SIZE = 10000 +# 200 is overhead of encryption (chosen manually) +COLUMN_DATA_SIZE = DATA_MAX_SIZE + 200 metadata = sa.MetaData() test_table = sa.Table('test', metadata, sa.Column('id', sa.Integer, primary_key=True), - sa.Column('data', sa.LargeBinary), - sa.Column('raw_data', sa.String), + sa.Column('data', sa.LargeBinary(length=COLUMN_DATA_SIZE)), + sa.Column('raw_data', sa.String(length=COLUMN_DATA_SIZE)), ) rollback_output_table = sa.Table('acra_rollback_output', metadata, @@ -52,6 +60,19 @@ zones = [] poison_record = None +master_key = None +ACRA_MASTER_KEY_VAR_NAME = 'ACRA_MASTER_KEY' +MASTER_KEY_PATH = 'master.key' + +CONFIG_UI_HTTP_PORT = 8022 +CONFIG_UI_AUTH_DB_PATH = 'auth.keys' +CONFIG_UI_BASIC_AUTH = dict( + user='test_user', + password='test_user_password' +) +CONFIG_UI_STATIC_PATH = 'cmd/acra_configui/static/' +CONFIG_HTTP_TIMEOUT = 3 + POISON_KEY_PATH = '.poison_key/poison_key' SETUP_SQL_COMMAND_TIMEOUT = 0.1 @@ -59,17 +80,31 @@ CONNECTION_FAIL_SLEEP = 0.1 SOCKET_CONNECT_TIMEOUT = 10 KILL_WAIT_TIMEOUT = 10 +CONNECT_TRY_COUNT = 3 +SQL_EXECUTE_TRY_COUNT = 5 -TEST_WITH_TLS = os.environ.get('TEST_TLS', 'on').lower() == 'on' +TEST_WITH_TLS = os.environ.get('TEST_TLS', 'off').lower() == 'on' PG_UNIX_HOST = '/tmp' + DB_USER = os.environ.get('TEST_DB_USER', 'postgres') DB_USER_PASSWORD = os.environ.get('TEST_DB_USER_PASSWORD', 'postgres') -SSLMODE = os.environ.get('TEST_SSL_MODE', 'allow') -connect_args = { - 'connect_timeout': SOCKET_CONNECT_TIMEOUT, - 'user': DB_USER, 'password': DB_USER_PASSWORD, - "options": "-c statement_timeout=1000", 'sslmode': SSLMODE} +SSLMODE = os.environ.get('TEST_SSL_MODE', 'require') +TEST_MYSQL = bool(os.environ.get('TEST_MYSQL', False)) +if TEST_MYSQL: + TEST_POSTGRESQL = False + DB_DRIVER = "mysql+pymysql" + TEST_MYSQL = True + connect_args = { + 'user': DB_USER, 'password': DB_USER_PASSWORD + } +else: + TEST_POSTGRESQL = True + DB_DRIVER = "postgresql" + connect_args = { + 'connect_timeout': SOCKET_CONNECT_TIMEOUT, + 'user': DB_USER, 'password': DB_USER_PASSWORD, + "options": "-c statement_timeout=1000", 'sslmode': SSLMODE} def stop_process(process): @@ -95,13 +130,30 @@ def stop_process(process): traceback.print_exc() -def get_connect_args(port=5432, sslmode='require', **kwargs): +def get_connect_args(port=5432, sslmode=None, **kwargs): args = connect_args.copy() - args['port'] = port - args['sslmode'] = sslmode + args['port'] = int(port) + if TEST_POSTGRESQL: + args['sslmode'] = sslmode if sslmode else SSLMODE args.update(kwargs) return args + +def get_master_key(): + """ + return master key in base64 format if generated or generate and return + """ + global master_key + if not master_key: + master_key = os.environ.get(ACRA_MASTER_KEY_VAR_NAME) + if not master_key: + subprocess.check_output([ + './acra_genkeys', '--master_key={}'.format(MASTER_KEY_PATH)]) + with open(MASTER_KEY_PATH, 'rb') as f: + master_key = b64encode(f.read()).decode('ascii') + return master_key + + def get_poison_record(): """generate one poison record for speed up tests and don't create subprocess for new records""" @@ -120,6 +172,13 @@ def create_client_keypair(name, only_server=False, only_client=False): args.append('-acraproxy') return subprocess.call(args, cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) +def manage_basic_auth_user(action, user_name, user_password): + args = ['./acra_genauth', '--{}'.format(action), + '--file={}'.format(CONFIG_UI_AUTH_DB_PATH), + '--user={}'.format(user_name), + '--password={}'.format(user_password)] + return subprocess.call(args, cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + def wait_connection(port, count=10, sleep=0.1): """try connect to 127.0.0.1:port and close connection @@ -153,49 +212,110 @@ def wait_unix_socket(socket_path, count=10, sleep=0.1): time.sleep(sleep) raise Exception("can't wait connection") +def get_unix_connection_string(port, dbname): + if TEST_MYSQL: + return get_postgresql_tcp_connection_string(port, dbname) + else: + return get_postgresql_unix_connection_string(port, dbname) def get_postgresql_unix_connection_string(port, dbname): - return 'postgresql+psycopg2:///{}?host={}'.format(dbname, PG_UNIX_HOST) + return '{}:///{}?host={}'.format(DB_DRIVER, dbname, PG_UNIX_HOST) def get_postgresql_tcp_connection_string(port, dbname): - return 'postgresql+psycopg2://127.0.0.1:{}/{}'.format(port, dbname) + return '{}://127.0.0.1:{}/{}'.format(DB_DRIVER, port, dbname) -def get_unix_connection_string(port): +def get_acra_unix_connection_string(port): return "unix://{}".format("{}/unix_socket_{}".format(PG_UNIX_HOST, port)) def get_proxy_connection_string(port): - return 'unix://{}/.s.PGSQL.{}'.format(PG_UNIX_HOST, port) + if TEST_MYSQL: + connection_string = get_postgresql_tcp_connection_string(port, '') + url = urlparse(connection_string) + return 'tcp://{}'.format(url.netloc) + else: + return 'unix://{}/.s.PGSQL.{}'.format(PG_UNIX_HOST, port) def get_tcp_connection_string(port): return 'tcp://127.0.0.1:{}'.format(port) def socket_path_from_connection_string(connection_string): - return connection_string.replace('unix://', '') + if '://' in connection_string: + return connection_string.split('://')[1] + else: + return connection_string def acra_api_connection_string(port): return "unix://{}".format("{}/acra_api_unix_socket_{}".format(PG_UNIX_HOST, port+1)) -BINARIES = ['acraproxy', 'acraserver', 'acra_addzone', 'acra_genkeys', - 'acra_genpoisonrecord', 'acra_rollback'] + + +DEFAULT_VERSION = '1.5.0' +DEFAULT_BUILD_ARGS = [] +ACRA_ROLLBACK_MIN_VERSION = "1.8.0" +Binary = collections.namedtuple( + 'Binary', ['name', 'from_version', 'build_args']) + + +BINARIES = [ + Binary(name='acraproxy', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS), + # compile with Test=true to disable golang tls client server verification + Binary(name='acraserver', from_version=DEFAULT_VERSION, + build_args=['-ldflags', '-X main.TestOnly=true']), + + Binary(name='acra_addzone', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS), + Binary(name='acra_genkeys', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS), + Binary(name='acra_genpoisonrecord', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS), + Binary(name='acra_rollback', from_version=ACRA_ROLLBACK_MIN_VERSION, + build_args=DEFAULT_BUILD_ARGS), + Binary(name='acra_genauth', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS), + Binary(name='acra_configui', from_version=DEFAULT_VERSION, + build_args=DEFAULT_BUILD_ARGS) +] def clean_binaries(): for i in BINARIES: try: - os.remove(i) + os.remove(i.name) except: pass +def clean_misc(): + try: + os.unlink('./{}'.format(CONFIG_UI_AUTH_DB_PATH)) + except: + pass + + PROCESS_CALL_TIMEOUT = 120 +def get_go_version(): + output = subprocess.check_output(['go', 'version']) + # example: go1.7.2 or go1.7 + version = re.search(r'go([\d.]+)', output.decode('utf-8')).group(1) + # convert to 3 part semver format + if version.count('.') < 2: + version = '{}.0'.format(version) + return version + def setUpModule(): global zones clean_binaries() + clean_misc() # build binaries builds = [ - ['go', 'build', 'github.com/cossacklabs/acra/cmd/{}'.format(binary)] + (binary.from_version, ['go', 'build'] + binary.build_args + ['github.com/cossacklabs/acra/cmd/{}'.format(binary.name)]) for binary in BINARIES ] - for build in builds: + go_version = get_go_version() + GREATER, EQUAL, LESS = (1, 0, -1) + for version, build in builds: + if semver.compare(go_version, version) == LESS: + continue # try to build 3 times with timeout build_count = 3 for i in range(build_count): @@ -207,6 +327,8 @@ def setUpModule(): raise continue + # must be before any call of key generators or forks of acra/proxy servers + os.environ.setdefault(ACRA_MASTER_KEY_VAR_NAME, get_master_key()) # first keypair for using without zones assert create_client_keypair('keypair1') == 0 assert create_client_keypair('keypair2') == 0 @@ -217,10 +339,12 @@ def setUpModule(): ['./acra_addzone'], cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT).decode('utf-8'))) socket.setdefaulttimeout(SOCKET_CONNECT_TIMEOUT) + def tearDownModule(): - import shutil shutil.rmtree('.acrakeys') clean_binaries() + clean_misc() + class ProcessStub(object): pid = 'stub' @@ -236,18 +360,29 @@ class BaseTestCase(unittest.TestCase): DB_HOST = os.environ.get('TEST_DB_HOST', '127.0.0.1') DB_NAME = os.environ.get('TEST_DB_NAME', 'postgres') DB_PORT = os.environ.get('TEST_DB_PORT', 5432) + DEBUG_LOG = os.environ.get('DEBUG_LOG', False) PROXY_PORT_1 = int(os.environ.get('TEST_PROXY_PORT', 9595)) PROXY_PORT_2 = PROXY_PORT_1 + 200 - PROXY_COMMAND_PORT_1 = int(os.environ.get('TEST_PROXY_COMMAND_PORT', 9595)) + PROXY_COMMAND_PORT_1 = int(os.environ.get('TEST_PROXY_COMMAND_PORT', 9696)) + CONFIG_UI_HTTP_PORT = int(os.environ.get('TEST_CONFIG_UI_HTTP_PORT', CONFIG_UI_HTTP_PORT)) # for debugging with manually runned acra server EXTERNAL_ACRA = False ACRA_PORT = int(os.environ.get('TEST_ACRA_PORT', 10003)) ACRA_BYTEA = 'hex_bytea' DB_BYTEA = 'hex' WHOLECELL_MODE = False + CONFIG_UI_AUTH_KEYS_PATH = os.environ.get('TEST_CONFIG_UI_AUTH_DB_PATH', CONFIG_UI_AUTH_DB_PATH) + CONFIG_UI_ACRA_SERVERR_PARAMS = dict( + db_host=DB_HOST, + db_port=DB_PORT, + commands_port=9090, + debug=DEBUG_LOG, + poisonscript="", + poisonshutdown=False, + zonemode=False + ) ZONE = False - DEBUG_LOG = False TEST_DATA_LOG = False TLS_ON = False maxDiff = None @@ -274,6 +409,20 @@ def fork(self, func): def wait_acra_connection(self, *args, **kwargs): return wait_unix_socket(*args, **kwargs) + def fork_configui(self, proxy_port: int, http_port: int): + args = [ + './acra_configui', + '-port={}'.format(http_port), + '-acra_host=127.0.0.1', + '-acra_port={}'.format(proxy_port), + '-static_path={}'.format(CONFIG_UI_STATIC_PATH) + ] + if self.DEBUG_LOG: + args.append('-d=true') + process = self.fork(lambda: subprocess.Popen(args)) + return process + + def fork_proxy(self, proxy_port: int, acra_port: int, client_id: str, commands_port: int=None, zone_mode: bool=False, check_connection: bool=True): acra_connection = self.get_acra_connection_string(acra_port) acra_api_connection = self.get_acra_api_connection_string(acra_port) @@ -302,7 +451,7 @@ def fork_proxy(self, proxy_port: int, acra_port: int, client_id: str, commands_p if self.DEBUG_LOG: args.append('-v=true') if zone_mode: - args.append('--zonemode=true') + args.append('--enable_http_api=true') if self.TLS_ON: args.append('--tls') args.append('--tls_ca=tests/server.crt') @@ -312,7 +461,10 @@ def fork_proxy(self, proxy_port: int, acra_port: int, client_id: str, commands_p process = self.fork(lambda: subprocess.Popen(args)) if check_connection: try: - wait_unix_socket(socket_path_from_connection_string(proxy_connection)) + if TEST_MYSQL: + wait_connection(proxy_port) + else: + wait_unix_socket(socket_path_from_connection_string(proxy_connection)) except: stop_process(process) raise @@ -321,7 +473,7 @@ def fork_proxy(self, proxy_port: int, acra_port: int, client_id: str, commands_p def get_acra_connection_string(self, port=None): if not port: port = self.ACRA_PORT - return get_unix_connection_string(port) + return get_acra_unix_connection_string(port) def get_acra_api_connection_string(self, port=None): if not port: @@ -338,6 +490,12 @@ def get_proxy_api_connection_string(self, port=None): port = self.PROXY_COMMAND_PORT_1 return get_proxy_connection_string(port) + def get_config_ui_connection_url(self): + return 'http://{}:{}'.format('localhost', CONFIG_UI_HTTP_PORT) + + def get_acraserver_bin_path(self): + return './acraserver' + def _fork_acra(self, acra_kwargs, popen_kwargs): connection_string = self.get_acra_connection_string() api_connection_string = self.get_acra_api_connection_string() @@ -350,6 +508,8 @@ def _fork_acra(self, acra_kwargs, popen_kwargs): args = { 'db_host': self.DB_HOST, 'db_port': self.DB_PORT, + # we doesn't need in tests waiting closing connections + 'close_connections_timeout': 0, self.ACRA_BYTEA: 'true', 'connection_string': connection_string, 'connection_api_string': api_connection_string, @@ -357,7 +517,8 @@ def _fork_acra(self, acra_kwargs, popen_kwargs): 'injectedcell': 'false' if self.WHOLECELL_MODE else 'true', 'd': 'true' if self.DEBUG_LOG else 'false', 'zonemode': 'true' if self.ZONE else 'false', - 'disable_http_api': 'false' if self.ZONE else 'true', + 'enable_http_api': 'true' if self.ZONE else 'true', + 'auth_keys': self.CONFIG_UI_AUTH_KEYS_PATH } if self.TLS_ON: args['tls'] = 'true' @@ -365,12 +526,15 @@ def _fork_acra(self, acra_kwargs, popen_kwargs): args['tls_cert'] = 'tests/server.crt' args['tls_ca'] = 'tests/server.crt' args['tls_sni'] = 'acraserver' + if TEST_MYSQL: + args['mysql'] = 'true' + args['postgresql'] = 'false' args.update(acra_kwargs) if not popen_kwargs: popen_kwargs = {} cli_args = ['--{}={}'.format(k, v) for k, v in args.items()] - process = self.fork(lambda: subprocess.Popen(['./acraserver'] + cli_args, + process = self.fork(lambda: subprocess.Popen([self.get_acraserver_bin_path()] + cli_args, **popen_kwargs)) try: self.wait_acra_connection(socket_path_from_connection_string(connection_string)) @@ -391,12 +555,12 @@ def setUp(self): self.acra = self.fork_acra() self.engine1 = sa.create_engine( - get_postgresql_unix_connection_string(self.PROXY_PORT_1, self.DB_NAME), connect_args=get_connect_args(port=self.PROXY_PORT_1)) + get_unix_connection_string(self.PROXY_PORT_1, self.DB_NAME), connect_args=get_connect_args(port=self.PROXY_PORT_1)) self.engine2 = sa.create_engine( - get_postgresql_unix_connection_string( + get_unix_connection_string( self.PROXY_PORT_2, self.DB_NAME), connect_args=get_connect_args(port=self.PROXY_PORT_2)) self.engine_raw = sa.create_engine( - 'postgresql://{}:{}/{}'.format(self.DB_HOST, self.DB_PORT, self.DB_NAME), + '{}://{}:{}/{}'.format(DB_DRIVER, self.DB_HOST, self.DB_PORT, self.DB_NAME), connect_args=connect_args) self.engines = [self.engine1, self.engine2, self.engine_raw] @@ -408,14 +572,18 @@ def setUp(self): # try with sleep if acra not up yet while True: try: - engine.execute( - "UPDATE pg_settings SET setting = '{}' " - "WHERE name = 'bytea_output'".format(self.DB_BYTEA)) + if TEST_MYSQL: + engine.execute( + "select 1;") + else: + engine.execute( + "UPDATE pg_settings SET setting = '{}' " + "WHERE name = 'bytea_output'".format(self.DB_BYTEA)) break except Exception: time.sleep(SETUP_SQL_COMMAND_TIMEOUT) count += 1 - if count == 3: + if count == SQL_EXECUTE_TRY_COUNT: raise except: self.tearDown() @@ -434,7 +602,7 @@ def tearDown(self): engine.dispose() def get_random_data(self): - size = random.randint(100, 10000) + size = random.randint(100, DATA_MAX_SIZE) return ''.join(random.choice(string.ascii_letters) for _ in range(size)) @@ -463,7 +631,7 @@ def log(self, acra_key_name, data, expected): 'zone_id': zones[0]['id'], 'poison_record': b64encode(get_poison_record()).decode('ascii'), } - )) + ), file=sys.stderr) class HexFormatTest(BaseTestCase): @@ -652,7 +820,20 @@ class ZoneEscapeFormatWholeCellTest(WholeCellMixinTest, ZoneEscapeFormatTest): class TestConnectionClosing(BaseTestCase): + class mysql_closing(contextlib.closing): + """ + extended contextlib.closing that add close() method that call close() + method of wrapped object + + Need to wrap pymysql.connection with own __enter__/__exit__ + implementation that will return connection instead of cursor (as do + pymysql.Connection.__enter__()) + """ + def close(self): + self.thing.close() + def setUp(self): + self.checkSkip() try: self.proxy_1 = self.fork_proxy( self.PROXY_PORT_1, self.ACRA_PORT, 'keypair1') @@ -662,9 +843,20 @@ def setUp(self): self.tearDown() raise - def get_connection(self): - return psycopg2.connect(host=PG_UNIX_HOST, **get_connect_args(port=self.PROXY_PORT_1)) + count = CONNECT_TRY_COUNT + while True: + try: + if TEST_MYSQL: + return TestConnectionClosing.mysql_closing( + pymysql.connect(**get_connect_args(port=self.PROXY_PORT_1))) + else: + return psycopg2.connect(host=PG_UNIX_HOST, **get_connect_args(port=self.PROXY_PORT_1)) + except: + count -= 1 + if count == 0: + raise + time.sleep(CONNECTION_FAIL_SLEEP) def tearDown(self): procs = [] @@ -675,8 +867,13 @@ def tearDown(self): stop_process(procs) def getActiveConnectionCount(self, cursor): - cursor.execute('select count(*) from pg_stat_activity;') - return int(cursor.fetchone()[0]) + if TEST_MYSQL: + query = "SHOW STATUS WHERE `variable_name` = 'Threads_connected';" + cursor.execute(query) + return int(cursor.fetchone()[1]) + else: + cursor.execute('select count(*) from pg_stat_activity;') + return int(cursor.fetchone()[0]) def getConnectionLimit(self, connection=None): created_connection = False @@ -684,72 +881,109 @@ def getConnectionLimit(self, connection=None): connection = self.get_connection() created_connection = True - cursor = connection.cursor() - cursor.execute('select setting from pg_settings where name=\'max_connections\';') - pg_max_connections = int(cursor.fetchone()[0]) - cursor.execute('select rolconnlimit from pg_roles where rolname = current_user;') - pg_rolconnlimit = int(cursor.fetchone()[0]) - cursor.close() - if created_connection: - connection.close() - if pg_rolconnlimit <= 0: - return pg_max_connections - return min(pg_max_connections, pg_rolconnlimit) + if TEST_MYSQL: + query = "SHOW VARIABLES WHERE `variable_name` = 'max_connections';" + with connection.cursor() as cursor: + cursor.execute(query) + return int(cursor.fetchone()[1]) + + else: + with connection.cursor() as cursor: + try: + cursor.execute('select setting from pg_settings where name=\'max_connections\';') + pg_max_connections = int(cursor.fetchone()[0]) + cursor.execute('select rolconnlimit from pg_roles where rolname = current_user;') + pg_rolconnlimit = int(cursor.fetchone()[0]) + cursor.close() + if pg_rolconnlimit <= 0: + return pg_max_connections + return min(pg_max_connections, pg_rolconnlimit) + except: + if created_connection: + connection.close() + raise def check_count(self, cursor, expected): # give a time to close connections via postgresql # because performance where tests will run not always constant, # we wait try_count times. in best case it will not need to sleep - try_count = 5 + try_count = SQL_EXECUTE_TRY_COUNT for i in range(try_count): try: self.assertEqual(self.getActiveConnectionCount(cursor), expected) + break except (AssertionError): if i == (try_count - 1): raise # some wait for closing. chosen manually time.sleep(1) - def testClosingConnections(self): - connection = self.get_connection() - - connection.autocommit = True - cursor = connection.cursor() - current_connection_count = self.getActiveConnectionCount(cursor) - - connection2 = self.get_connection() - self.assertEqual(self.getActiveConnectionCount(cursor), - current_connection_count+1) - connection_limit = self.getConnectionLimit(connection) - connections = [connection2] - with self.assertRaises(psycopg2.OperationalError) as context_manager: - for i in range(connection_limit): - connections.append(self.get_connection()) - exception = context_manager.exception - # exception doesn't has any related code, only text messages - correct_messages = [ - 'FATAL: too many connections for role', - 'FATAL: sorry, too many clients already'] - is_correct_exception_message = False - for message in correct_messages: - if message in exception.args[0]: - is_correct_exception_message = True - break - self.assertTrue(is_correct_exception_message) + def checkConnectionLimit(self, connection_limit): + connections = [] + try: + exception = None + try: + for i in range(connection_limit): + connections.append(self.get_connection()) + except Exception as exc: + exception = exc + + self.assertIsNotNone(exception) + + is_correct_exception_message = False + if TEST_MYSQL: + exception_type = pymysql.err.OperationalError + correct_messages = [ + 'Too many connections' + ] + for message in correct_messages: + if exception.args[0] in [1203, 1040] and message in exception.args[1]: + is_correct_exception_message = True + break + else: + exception_type = psycopg2.OperationalError + # exception doesn't has any related code, only text messages + correct_messages = [ + 'FATAL: too many connections for role', + 'FATAL: sorry, too many clients already', + 'FATAL: remaining connection slots are reserved for non-replication superuser connections' + ] + for message in correct_messages: + if message in exception.args[0]: + is_correct_exception_message = True + break - for conn in connections: - conn.close() + self.assertIsInstance(exception, exception_type) + self.assertTrue(is_correct_exception_message) + except: + for connection in connections: + connection.close() + raise + return connections + + def testClosingConnectionsWithDB(self): + with self.get_connection() as connection: + connection.autocommit = True + with connection.cursor() as cursor: + current_connection_count = self.getActiveConnectionCount(cursor) - self.check_count(cursor, current_connection_count) + with self.get_connection(): + self.assertEqual(self.getActiveConnectionCount(cursor), + current_connection_count+1) + connection_limit = self.getConnectionLimit(connection) - # try create new connection - connection2 = self.get_connection() - self.check_count(cursor, current_connection_count + 1) + created_connections = self.checkConnectionLimit( + connection_limit) + for conn in created_connections: + conn.close() - connection2.close() - self.check_count(cursor, current_connection_count) - cursor.close() - connection.close() + self.check_count(cursor, current_connection_count) + + # try create new connection + with self.get_connection(): + self.check_count(cursor, current_connection_count + 1) + + self.check_count(cursor, current_connection_count) class TestKeyNonExistence(BaseTestCase): @@ -857,6 +1091,7 @@ def test_without_acraserver_public(self): class BasePoisonRecordTest(BaseTestCase): SHUTDOWN = True + TEST_DATA_LOG = True def setUp(self): super(BasePoisonRecordTest, self).setUp() @@ -879,23 +1114,33 @@ class TestPoisonRecordShutdown(BasePoisonRecordTest): def testShutdown(self): row_id = self.get_random_id() + data = get_poison_record() self.engine1.execute( test_table.insert(), - {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) + {'id': row_id, 'data': data, 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): - self.engine1.execute( + result = self.engine1.execute( sa.select([test_table]) .where(test_table.c.id == row_id)) + row = result.fetchone() + if row['data'] == data: + self.fail("unexpected response") def testShutdown2(self): """check working poison record callback on full select""" row_id = self.get_random_id() + data = get_poison_record() self.engine1.execute( test_table.insert(), - {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) + {'id': row_id, 'data': data, 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): - self.engine1.execute( + result = self.engine1.execute( sa.select([test_table])) + rows = result.fetchall() + for row in rows: + if row['id'] == row_id and row['data'] == data: + self.fail("unexpected response") + def testShutdown3(self): """check working poison record callback on full select inside another data""" @@ -905,8 +1150,12 @@ def testShutdown3(self): test_table.insert(), {'id': row_id, 'data': data, 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): - self.engine1.execute( + result = self.engine1.execute( sa.select([test_table])) + rows = result.fetchall() + for row in rows: + if row['id'] == row_id and row['data'] == data: + self.fail("unexpected response") class TestShutdownPoisonRecordWithZone(TestPoisonRecordShutdown): @@ -922,9 +1171,10 @@ def testShutdown(self): {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): zone = zones[0]['id'].encode('ascii') - self.engine1.execute( + result = self.engine1.execute( sa.select([sa.cast(zone, BYTEA), test_table]) .where(test_table.c.id == row_id)) + print(result.fetchall()) def testShutdown2(self): """check callback with select by id and without zone""" @@ -933,9 +1183,10 @@ def testShutdown2(self): test_table.insert(), {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): - self.engine1.execute( + result = self.engine1.execute( sa.select([test_table]) .where(test_table.c.id == row_id)) + print(result.fetchall()) def testShutdown3(self): """check working poison record callback on full select""" @@ -944,8 +1195,9 @@ def testShutdown3(self): test_table.insert(), {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) with self.assertRaises(DatabaseError): - self.engine1.execute( + result = self.engine1.execute( sa.select([test_table])) + print(result.fetchall()) def testShutdown4(self): """check working poison record callback on full select inside another data""" @@ -1002,14 +1254,14 @@ def testNoDetect(self): {'id': row_id, 'data': get_poison_record(), 'raw_data': 'poison_record'}) result = self.engine1.execute(test_table.select()) result.fetchall() - # super() tearDown without killink acra + # super() tearDown without killing acra super(TestNoCheckPoisonRecord, self).tearDown() try: out, er_ = self.acra.communicate(timeout=1) except subprocess.TimeoutExpired: pass - self.assertNotIn(b'Debug: check poison records', out) + self.assertNotIn(b'Check poison records', out) class TestNoCheckPoisonRecordWithZone(TestNoCheckPoisonRecord): @@ -1027,6 +1279,7 @@ class TestNoCheckPoisonRecordWithZoneWholeCell(TestNoCheckPoisonRecordWithZone): class TestCheckLogPoisonRecord(AcraCatchLogsMixin, BasePoisonRecordTest): SHUTDOWN = True DEBUG_LOG = True + TEST_DATA_LOG = True def setUp(self): self.poison_script_file = NamedTemporaryFile('w') @@ -1048,14 +1301,14 @@ def testDetect(self): with self.assertRaises(DatabaseError): self.engine1.execute(test_table.select()) finally: - # super() tearDown without killink acra + # super() tearDown without killing acra super(TestCheckLogPoisonRecord, self).tearDown() try: out, _ = self.acra.communicate(timeout=1) except subprocess.TimeoutExpired: pass - self.assertIn(b'check poison records', out) + self.assertIn(b'Check poison records', out) class TestKeyStorageClearing(BaseTestCase): @@ -1069,14 +1322,14 @@ def setUp(self): zone_mode=True) if not self.EXTERNAL_ACRA: self.acra = self.fork_acra( - zonemode='true', disable_http_api='false') + zonemode='true', enable_http_api='true') self.engine1 = sa.create_engine( - get_postgresql_unix_connection_string(self.PROXY_PORT_1, self.DB_NAME), + get_unix_connection_string(self.PROXY_PORT_1, self.DB_NAME), connect_args=get_connect_args(port=self.PROXY_PORT_1)) self.engine_raw = sa.create_engine( - 'postgresql://{}:{}/{}'.format(self.DB_HOST, self.DB_PORT, self.DB_NAME), + '{}://{}:{}/{}'.format(DB_DRIVER, self.DB_HOST, self.DB_PORT, self.DB_NAME), connect_args=connect_args) self.engines = [self.engine1, self.engine_raw] @@ -1122,18 +1375,60 @@ def test_clearing(self): class TestAcraRollback(BaseTestCase): DATA_COUNT = 5 + def checkSkip(self): + super(TestAcraRollback, self).checkSkip() + go_version = get_go_version() + GREATER, EQUAL, LESS = (1, 0, -1) + if semver.compare(go_version, ACRA_ROLLBACK_MIN_VERSION) == LESS: + self.skipTest("not supported go version") + def setUp(self): + self.checkSkip() self.engine_raw = sa.create_engine( - 'postgresql://{}:{}/{}'.format(self.DB_HOST, self.DB_PORT, - self.DB_NAME), + '{}://{}:{}/{}'.format(DB_DRIVER, self.DB_HOST, self.DB_PORT, + self.DB_NAME), connect_args=connect_args) self.output_filename = 'acra_rollback_output.txt' rollback_output_table.create(self.engine_raw, checkfirst=True) - if self.TLS_ON: + if TEST_WITH_TLS: self.sslmode='require' else: self.sslmode='disable' + if TEST_MYSQL: + # https://github.com/go-sql-driver/mysql/ + connection_string = "{user}:{password}@tcp({host}:{port})/{dbname}".format( + user=DB_USER, password=DB_USER_PASSWORD, dbname=self.DB_NAME, + port=self.DB_PORT, host=self.DB_HOST + ) + + # https://github.com/ziutek/mymysql + # connection_string = "tcp:{host}:{port}*{dbname}/{user}/{password}".format( + # user=DB_USER, password=DB_USER_PASSWORD, dbname=self.DB_NAME, + # port=self.DB_PORT, host=self.DB_HOST + # ) + else: + connection_string = ( + 'dbname={dbname} user={user} ' + 'sslmode={sslmode} password={password} host={host} ' + 'port={port}').format( + sslmode=self.sslmode, dbname=self.DB_NAME, + user=DB_USER, port=self.DB_PORT, + password=DB_USER_PASSWORD, host=self.DB_HOST + ) + + if TEST_MYSQL: + self.placeholder = "?" + DB_ARGS = ['--mysql'] + else: + self.placeholder = "$1" + DB_ARGS = ['--postgresql'] + + self.default_rollback_args = [ + '--client_id=keypair1', + '--connection_string={}'.format(connection_string), + '--output_file={}'.format(self.output_filename), + ] + DB_ARGS def tearDown(self): try: @@ -1145,6 +1440,18 @@ def tearDown(self): if os.path.exists(self.output_filename): os.remove(self.output_filename) + def run_rollback(self, extra_args): + args = ['./acra_rollback'] + self.default_rollback_args + extra_args + try: + subprocess.check_call( + args, cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + except subprocess.CalledProcessError as exc: + if exc.stderr: + print(exc.stderr, file=sys.stderr) + else: + print(exc.stdout, file=sys.stdout) + raise + def test_without_zone_to_file(self): keyname = 'keypair1_storage' with open('.acrakeys/{}.pub'.format(keyname), 'rb') as f: @@ -1160,19 +1467,12 @@ def test_without_zone_to_file(self): } rows.append(row) self.engine_raw.execute(test_table.insert(), rows) - subprocess.check_call( - ['./acra_rollback', '--client_id=keypair1', - '--connection_string=dbname={dbname} user={user} ' - 'sslmode={sslmode} password={password} host={host} ' - 'port={port}'.format( - sslmode=self.sslmode, dbname=self.DB_NAME, - user=DB_USER, port=self.DB_PORT, - password=DB_USER_PASSWORD, host=self.DB_HOST), - '--output_file={}'.format(self.output_filename), - '--select=select data from {};'.format(test_table.name), - '--insert=insert into {} values($1);'.format( - rollback_output_table.name)], - cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + args = [ + '--select=select data from {};'.format(test_table.name), + '--insert=insert into {} values({});'.format( + rollback_output_table.name, self.placeholder) + ] + self.run_rollback(args) # execute file with open(self.output_filename, 'r') as f: @@ -1199,22 +1499,19 @@ def test_with_zone_to_file(self): } rows.append(row) self.engine_raw.execute(test_table.insert(), rows) - - subprocess.check_call( - ['./acra_rollback', '--client_id=keypair1', - '--connection_string=dbname={dbname} user={user} ' - 'sslmode={sslmode} ' - 'password={password} host={host} port={port}'.format( - dbname=self.DB_NAME, user=DB_USER, port=self.DB_PORT, - sslmode=self.sslmode, - password=DB_USER_PASSWORD, host=self.DB_HOST), - '--output_file={}'.format(self.output_filename), - '--select=select \'{id}\'::bytea, data from {table};'.format( - id=zones[0]['id'], table=test_table.name), + if TEST_MYSQL: + select_query = '--select=select \'{id}\', data from {table};'.format( + id=zones[0]['id'], table=test_table.name) + else: + select_query = '--select=select \'{id}\'::bytea, data from {table};'.format( + id=zones[0]['id'], table=test_table.name) + args = [ + select_query, '--zonemode=true', - '--insert=insert into {} values($1);'.format( - rollback_output_table.name)], - cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + '--insert=insert into {} values({});'.format( + rollback_output_table.name, self.placeholder) + ] + self.run_rollback(args) # execute file with open(self.output_filename, 'r') as f: @@ -1243,19 +1540,13 @@ def test_without_zone_execute(self): rows.append(row) self.engine_raw.execute(test_table.insert(), rows) - subprocess.check_call( - ['./acra_rollback', '--client_id=keypair1', - '--connection_string=dbname={dbname} user={user} ' - 'sslmode={sslmode} ' - 'password={password} host={host} port={port}'.format( - sslmode=self.sslmode, - dbname=self.DB_NAME, user=DB_USER, port=self.DB_PORT, - password=DB_USER_PASSWORD, host=self.DB_HOST), - '--execute=true', - '--select=select data from {};'.format(test_table.name), - '--insert=insert into {} values($1);'.format( - rollback_output_table.name)], - cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + args = [ + '--execute=true', + '--select=select data from {};'.format(test_table.name), + '--insert=insert into {} values({});'.format( + rollback_output_table.name, self.placeholder) + ] + self.run_rollback(args) source_data = set([i['raw_data'].encode('ascii') for i in rows]) result = self.engine_raw.execute(rollback_output_table.select()) @@ -1278,21 +1569,20 @@ def test_with_zone_execute(self): rows.append(row) self.engine_raw.execute(test_table.insert(), rows) - subprocess.check_call( - ['./acra_rollback', '--client_id=keypair1', - '--connection_string=dbname={dbname} user={user} ' - 'password={password} host={host} port={port} ' - 'sslmode={sslmode}'.format( - sslmode=self.sslmode, - dbname=self.DB_NAME, user=DB_USER, port=self.DB_PORT, - password=DB_USER_PASSWORD, host=self.DB_HOST), - '--execute=true', - '--select=select \'{id}\'::bytea, data from {table};'.format( - id=zones[0]['id'], table=test_table.name), - '--zonemode=true', - '--insert=insert into {} values($1);'.format( - rollback_output_table.name)], - cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + if TEST_MYSQL: + select_query = '--select=select \'{id}\', data from {table};'.format( + id=zones[0]['id'], table=test_table.name) + else: + select_query = '--select=select \'{id}\'::bytea, data from {table};'.format( + id=zones[0]['id'], table=test_table.name) + args = [ + '--execute=true', + select_query, + '--zonemode=true', + '--insert=insert into {} values({});'.format( + rollback_output_table.name, self.placeholder) + ] + self.run_rollback(args) source_data = set([i['raw_data'].encode('ascii') for i in rows]) result = self.engine_raw.execute(rollback_output_table.select()) @@ -1307,6 +1597,80 @@ def test_only_alpha_client_id(self): self.assertEqual(create_client_keypair(POISON_KEY_PATH), 1) +class TestAcraConfigUIGenAuth(unittest.TestCase): + def testUIGenAuth(self): + self.assertEqual(manage_basic_auth_user('set', 'test', 'test'), 0) + self.assertEqual(manage_basic_auth_user('set', CONFIG_UI_BASIC_AUTH['user'], CONFIG_UI_BASIC_AUTH['password']), 0) + self.assertEqual(manage_basic_auth_user('remove', 'test', 'test'), 0) + self.assertEqual(manage_basic_auth_user('remove', 'test_unknown', 'test_unknown'), 1) + + +class TestAcraConfigUIWeb(BaseTestCase): + def setUp(self): + try: + self.proxy_1 = self.fork_proxy( + self.PROXY_PORT_1, self.ACRA_PORT, 'keypair1', zone_mode=True, commands_port=self.PROXY_COMMAND_PORT_1) + self.acra = self.fork_acra(zonemode='true', enable_http_api='true') + self.configui = self.fork_configui(proxy_port=self.PROXY_COMMAND_PORT_1, http_port=self.CONFIG_UI_HTTP_PORT) + except Exception: + self.tearDown() + raise + + def tearDown(self): + try: + os.unlink('configs/acraserver.yaml') + except Exception as e: + print(e) + stop_process([self.configui]) + try: + subprocess.call(['killall', '--signal=SIGTERM', 'acraserver'], cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + except Exception as e: + print('SIGTERM->acraserver error: {}'.format(e)) + try: + subprocess.call(['killall', '--signal=SIGKILL', 'acraserver'], cwd=os.getcwd(), timeout=PROCESS_CALL_TIMEOUT) + except Exception as e: + print('SIGKILL->acraserver error: {}'.format(e)) + super(TestAcraConfigUIWeb, self).tearDown() + + def testAuthAndSubmitSettings(self): + import requests + import uuid + from requests.auth import HTTPBasicAuth + # test wrong auth + req = requests.post( + self.get_config_ui_connection_url(), data={}, timeout=CONFIG_HTTP_TIMEOUT, + auth=HTTPBasicAuth('wrong_user_name', 'wrong_password')) + self.assertEqual(req.status_code, 401) + req.close() + + # test correct auth + req = requests.post( + self.get_config_ui_connection_url(), data={}, timeout=CONFIG_HTTP_TIMEOUT, + auth=HTTPBasicAuth(CONFIG_UI_BASIC_AUTH['user'], CONFIG_UI_BASIC_AUTH['password'])) + self.assertEqual(req.status_code, 200) + req.close() + + # test submit settings + settings = self.CONFIG_UI_ACRA_SERVERR_PARAMS + settings['poisonscript'] = str(uuid.uuid4()) + print(settings) + req = requests.post( + "{}/acraserver/submit_setting".format(self.get_config_ui_connection_url()), + data=settings, + timeout=CONFIG_HTTP_TIMEOUT, + auth=HTTPBasicAuth(CONFIG_UI_BASIC_AUTH['user'], CONFIG_UI_BASIC_AUTH['password'])) + self.assertEqual(req.status_code, 200) + req.close() + + # check for new config after acraserver's graceful restart + req = requests.post( + self.get_config_ui_connection_url(), data={}, timeout=CONFIG_HTTP_TIMEOUT, + auth=HTTPBasicAuth(CONFIG_UI_BASIC_AUTH['user'], CONFIG_UI_BASIC_AUTH['password'])) + self.assertEqual(req.status_code, 200) + self.assertIn(settings['poisonscript'], req.text) + req.close() + + class SSLPostgresqlConnectionTest(HexFormatTest): def get_acra_connection_string(self): return get_tcp_connection_string(self.ACRA_PORT) @@ -1315,7 +1679,7 @@ def wait_acra_connection(self, *args, **kwargs): wait_connection(self.ACRA_PORT) def checkSkip(self): - if not TEST_WITH_TLS: + if not (TEST_WITH_TLS and TEST_POSTGRESQL): self.skipTest("running tests without TLS") def setUp(self): @@ -1326,11 +1690,13 @@ def setUp(self): try: if not self.EXTERNAL_ACRA: self.acra = self.fork_acra( - tls_key='tests/server.key', tls_cert='tests/server.crt', no_encryption=True, client_id='keypair1') + tls_key='tests/server.key', tls_cert='tests/server.crt', + tls_ca='tests/server.crt', + no_encryption=True, client_id='keypair1') self.engine1 = sa.create_engine( get_postgresql_tcp_connection_string(self.ACRA_PORT, self.DB_NAME), connect_args=get_connect_args(port=self.ACRA_PORT)) self.engine_raw = sa.create_engine( - 'postgresql://{}:{}/{}'.format(self.DB_HOST, self.DB_PORT, self.DB_NAME), + '{}://{}:{}/{}'.format(DB_DRIVER, self.DB_HOST, self.DB_PORT, self.DB_NAME), connect_args=get_connect_args(self.DB_PORT)) # test case from HexFormatTest expect two engines with different client_id but here enough one and # raw connection @@ -1352,7 +1718,7 @@ def setUp(self): except Exception: time.sleep(SETUP_SQL_COMMAND_TIMEOUT) count += 1 - if count == 3: + if count == SQL_EXECUTE_TRY_COUNT: raise except: self.tearDown() @@ -1365,10 +1731,14 @@ def tearDown(self): try: self.engine_raw.execute('delete from test;') + except: + traceback.print_exc() + + try: for engine in self.engines: engine.dispose() except: - pass + traceback.print_exc() class SSLPostgresqlConnectionWithZoneTest(ZoneHexFormatTest, @@ -1393,5 +1763,89 @@ class TLSBetweenProxyAndServerWithZonesTest(ZoneHexFormatTest, pass +class SSLMysqlConnectionTest(HexFormatTest): + def get_acra_connection_string(self): + return get_tcp_connection_string(self.ACRA_PORT) + + def wait_acra_connection(self, *args, **kwargs): + wait_connection(self.ACRA_PORT) + + def checkSkip(self): + if not (TEST_WITH_TLS and TEST_MYSQL): + self.skipTest("running tests without TLS") + + def setUp(self): + self.checkSkip() + """don't fork proxy, connect directly to acra, use ssl for connections and tcp protocol on acra side + because postgresql support tls only over tcp + """ + try: + if not self.EXTERNAL_ACRA: + self.acra = self.fork_acra( + tls_key='tests/server.key', + tls_cert='tests/server.crt', + tls_ca='tests/server.crt', + tls_sni="acraserver", + no_encryption=True, client_id='keypair1') + driver_to_acraserver_ssl_settings = { + 'ca': 'tests/server.crt', + 'cert': 'tests/client.crt', + 'key': 'tests/client.key', + 'check_hostname': False + } + self.engine_raw = sa.create_engine( + '{}://{}:{}/{}'.format(DB_DRIVER, self.DB_HOST, + self.DB_PORT, self.DB_NAME), + # don't provide any client's certificates to driver that connects + # directly to mysql to avoid verifying by mysql server + connect_args=get_connect_args(self.DB_PORT, ssl={'ca': None})) + + self.engine1 = sa.create_engine( + get_postgresql_tcp_connection_string(self.ACRA_PORT, self.DB_NAME), + connect_args=get_connect_args( + port=self.ACRA_PORT, ssl=driver_to_acraserver_ssl_settings)) + + # test case from HexFormatTest expect two engines with different + # client_id but here enough one and raw connection + self.engine2 = self.engine_raw + + self.engines = [self.engine1, self.engine_raw] + + metadata.create_all(self.engine_raw) + self.engine_raw.execute('delete from test;') + for engine in self.engines: + count = 0 + # try with sleep if acra not up yet + while True: + try: + engine.execute("select 1") + break + except Exception: + time.sleep(SETUP_SQL_COMMAND_TIMEOUT) + count += 1 + if count == SQL_EXECUTE_TRY_COUNT: + raise + except: + self.tearDown() + raise + + def tearDown(self): + if not self.EXTERNAL_ACRA: + if hasattr(self, 'acra'): + stop_process(self.acra) + + try: + self.engine_raw.execute('delete from test;') + for engine in self.engines: + engine.dispose() + except: + pass + + +class SSLMysqlConnectionWithZoneTest(ZoneHexFormatTest, + SSLMysqlConnectionTest): + pass + + if __name__ == '__main__': unittest.main() diff --git a/utils/logging.go b/utils/logging.go index 8a315e014..3354afcd5 100644 --- a/utils/logging.go +++ b/utils/logging.go @@ -15,6 +15,8 @@ package utils import "fmt" +// TODO: move to /logging and refactor everything + // function for standartizing custom messages with messages from error func ErrorMessage(msg string, err error) string { return fmt.Sprintf("%v (error message - %v)", msg, err) diff --git a/utils/version.go b/utils/version.go new file mode 100644 index 000000000..9beae3973 --- /dev/null +++ b/utils/version.go @@ -0,0 +1,3 @@ +package utils + +var VERSION string = "0.77.0" // change on current during build diff --git a/zone/zone_id_matcher_test.go b/zone/zone_id_matcher_test.go index ec3c011fc..acf55c538 100644 --- a/zone/zone_id_matcher_test.go +++ b/zone/zone_id_matcher_test.go @@ -48,6 +48,9 @@ func (storage *TestKeyStore) GenerateDataEncryptionKeys(id []byte) error { retur func (storage *TestKeyStore) GetServerDecryptionPrivateKey(id []byte) (*keys.PrivateKey, error) { return nil, nil } +func (keystore *TestKeyStore) GetAuthKey(remove bool) ([]byte, error) { + return nil, nil +} func (storage *TestKeyStore) GetPoisonKeyPair() (*keys.Keypair, error) { return nil, nil } func testZoneIdMatcher(t *testing.T) {