Skip to content

Commit

Permalink
Merge branch 'master' into stable
Browse files Browse the repository at this point in the history
  • Loading branch information
vixentael committed Apr 13, 2018
2 parents 95d61cc + aadae57 commit d619a20
Show file tree
Hide file tree
Showing 114 changed files with 8,209 additions and 1,250 deletions.
22 changes: 17 additions & 5 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,32 @@ jobs:
# postgresql image with ssl support
- image: nimbustech/postgres-ssl:9.5
environment:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: test
POSTGRES_USER: test
POSTGRES_DB: test
# use the same credentials for mysql db as for postgresql (which support was added first)
# has latest tag on 2018.03.29
- image: mysql:5.7.21
environment:
MYSQL_DATABASE: test
MYSQL_USER: test
MYSQL_PASSWORD: test
MYSQL_ROOT_PASSWORD: root
environment:
GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
FILEPATH_ERROR_FLAG: /tmp/test_fail
VERSIONS: 1.6 1.6.4 1.7 1.7.5 1.8 1.9.4 1.10
VERSIONS: 1.7 1.7.6 1.8 1.8.7 1.9.4 1.10
TEST_DB_USER: test
TEST_DB_USER_PASSWORD: test
TEST_DB_NAME: test
steps:
# prepare
- run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync
- run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync psmisc
- run: cd $HOME && git clone https://github.com/cossacklabs/themis && cd themis && sudo make install
- run: cd $HOME && for version in $VERSIONS; do mkdir go_root_$version; cd go_root_$version; wget https://storage.googleapis.com/golang/go$version.linux-amd64.tar.gz; tar xf go$version.linux-amd64.tar.gz; cd -; done
- checkout
- run: cd $HOME && for version in $VERSIONS; do mkdir -p go_path_$version/src/github.com/cossacklabs/themis/gothemis; mkdir -p go_path_$version/src/github.com/cossacklabs/acra; rsync -auv $HOME/themis/gothemis/ go_path_$version/src/github.com/cossacklabs/themis/gothemis; rsync -auv $HOME/project/ go_path_$version/src/github.com/cossacklabs/acra; done
- run: cd $HOME && for version in $VERSIONS; do GOROOT=$HOME/go_root_$version/go PATH=$GOROOT/bin/:$PATH GOPATH=$HOME/go_path_$version go get github.com/cossacklabs/acra/...; done
- run: cd $HOME && for version in $VERSIONS; do GOROOT=$HOME/go_root_$version/go PATH=$GOROOT/bin/:$PATH GOPATH=$HOME/go_path_$version go get -d github.com/cossacklabs/acra/...; done
- run: pip3 install -r $HOME/project/tests/requirements.txt
- run: sudo ldconfig
# testing
Expand Down
68 changes: 52 additions & 16 deletions .circleci/integration.sh
Original file line number Diff line number Diff line change
@@ -1,23 +1,59 @@
#!/usr/bin/env bash

export TEST_ACRA_PORT=6000
export TEST_PROXY_PORT=7000
export TEST_PROXY_COMMAND_PORT=8000
cd $HOME/project
for version in $VERSIONS; do
export TEST_ACRA_PORT=$(expr ${TEST_ACRA_PORT} + 1);
export TEST_PROXY_PORT=$(expr ${TEST_PROXY_PORT} + 1);
export TEST_PROXY_COMMAND_PORT=$(expr ${TEST_PROXY_COMMAND_PORT} + 1);
export GOROOT=$HOME/go_root_$version/go;
export PATH=$GOROOT/bin/:$PATH;
export GOPATH=$HOME/go_path_$version;

export TEST_TLS=on
python3 tests/test.py;
if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG";
fi

export TEST_TLS=off
python3 tests/test.py;
if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG";
fi
echo "-------------------- Testing Go version $version"

export TEST_ACRA_PORT=$(expr ${TEST_ACRA_PORT} + 1);
export TEST_PROXY_PORT=$(expr ${TEST_PROXY_PORT} + 1);
export TEST_PROXY_COMMAND_PORT=$(expr ${TEST_PROXY_COMMAND_PORT} + 1);
export GOROOT=$HOME/go_root_$version/go;
export PATH=$GOROOT/bin/:$PATH;
export GOPATH=$HOME/go_path_$version;

# setup postgresql credentials
#export TEST_DB_USER=${POSTGRES_USER}
#export TEST_DB_USER_PASSWORD=${POSTGRES_PASSWORD}
#export TEST_DB_NAME=postgres
export TEST_DB_PORT=5432
unset TEST_MYSQL

export TEST_TLS=on

echo "-------------------- Testing POSTGRES with TEST_TLS=on"

python3 tests/test.py -v;
if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG";
fi

export TEST_TLS=off

echo "-------------------- Testing POSTGRES with TEST_TLS=off"
python3 tests/test.py -v;
if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG";
fi

# setup mysql credentials
#export TEST_DB_USER=${MYSQL_USER}
#export TEST_DB_USER_PASSWORD=${MYSQL_PASSWORD}
#export TEST_DB_NAME=${MYSQL_DATABASE}
export TEST_DB_PORT=3306
export TEST_MYSQL=true


echo "-------------------- Testing TEST_MYSQL with TEST_TLS=off"
export TEST_TLS=off
python3 tests/test.py -v;
if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG";
fi

echo "-------------------- Testing TEST_MYSQL with TEST_TLS=on"
export TEST_TLS=on
python3 tests/test.py -v;
if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG";
fi

done
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,5 @@ ENV/

# Rope project settings
.ropeproject

cmd/acra_configui/auth.keys
87 changes: 87 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,92 @@
# Acra ChangeLog

## [0.77.0](https://github.com/cossacklabs/acra/releases/tag/0.77), April 13th 2018


_Core_:

- **MySQL databases**

- Added support for MySQL: now you can connect Acra to MySQL databases. Works with any SSL mode: `require`, `allow`, `disable`.
- Tested and supported on: MySQL ([#155](https://github.com/cossacklabs/acra/pull/155), [#140](https://github.com/cossacklabs/acra/pull/140)).

> Note: Prepared statements are not supported yet, but this feature is coming soon!

Read about the new configurations on the [AcraServer](https://github.com/cossacklabs/acra/wiki/How-AcraServer-works) documentation page.

- **Keeping keys in secret**

- Added encryption for the keys' folder: private keys are now symmetrically encrypted by `master_key` ([#143](https://github.com/cossacklabs/acra/pull/143)) for storage.
- Added ability to generate public/private keys in the separate folders ([#148](https://github.com/cossacklabs/acra/pull/148), [#142](https://github.com/cossacklabs/acra/pull/142)).

Read more about the current changes in [key management here](https://github.com/cossacklabs/acra/wiki/Key-Management).

- **Filtering requests for MySQL**

- Added firewall component named [AcraCensor](https://github.com/cossacklabs/acra/wiki/acracensor) to handle MySQL queries. <br/>
You can provide a list of restricted or allowed tables, columns, and exact queries to handle. AcraCensor will pass the allowed queries and return error on forbidden ones. Rules are configured and stored in `yaml` file. Each request is logged in real time. Moreover, all the queries and their states are logged into a separate log file. ([#151](https://github.com/cossacklabs/acra/pull/151), [#138](https://github.com/cossacklabs/acra/pull/138), [#136](https://github.com/cossacklabs/acra/pull/136), [#132](https://github.com/cossacklabs/acra/pull/132), [#125](https://github.com/cossacklabs/acra/pull/125), [#108](https://github.com/cossacklabs/acra/pull/108)).<br/>

See a detailed description of AcraCensor on the corresponding [AcraCensor documentation page](https://github.com/cossacklabs/acra/wiki/acracensor).

- **Web Config UI**

- Added lightweight HTTP [web server](https://github.com/cossacklabs/acra/wiki/AcraConfigUI) for managing AcraServer's certain configuration options.<br/>
You can update the proxy address and port, database address and port, handling of Zone mode and poison records. On saving new configuration, `acraserver` will gracefully restart and use these settings automatically. The access to thiw web page is restricted using basic auth. ([#153](https://github.com/cossacklabs/acra/pull/153), [#141](https://github.com/cossacklabs/acra/pull/141), [#123](https://github.com/cossacklabs/acra/pull/123), [#111](https://github.com/cossacklabs/acra/pull/111)).<br/>

See the interface screenshot and detailed instructions at [Acra Config UI](https://github.com/cossacklabs/acra/wiki/AcraConfigUI) page.


- **Logging**
- Added support of new logging formats: plaintext, [CEF](https://kc.mcafee.com/resources/sites/MCAFEE/content/live/CORP_KNOWLEDGEBASE/78000/KB78712/en_US/CEF_White_Paper_20100722.pdf), and json.<br/>
Logging mode and verbosity level is configured for AcraServer, AcraProxy, and AcraConfigUI in the corresponding `yaml` files. Log messages were slightly improved, custom error codes were added (which we believe will help to understand and debug any issues) ([#135](https://github.com/cossacklabs/acra/pull/135), [#126](https://github.com/cossacklabs/acra/pull/126), [#110](https://github.com/cossacklabs/acra/pull/110)).

Read more about the log analysis at [Logging](https://github.com/cossacklabs/acra/wiki/Logging) page.


- **Tests**

- Added many new integartion tests, fixed stability and handling of more complicated use-cases ([#150](https://github.com/cossacklabs/acra/pull/150), [#147](https://github.com/cossacklabs/acra/pull/147), [#137](https://github.com/cossacklabs/acra/pull/137), [#117](https://github.com/cossacklabs/acra/pull/117), [#116](https://github.com/cossacklabs/acra/pull/116), [#115](https://github.com/cossacklabs/acra/pull/115)).


_Infrastructure_:

- **Docker support**

- Added Docker Container for every main component: `AcraServer`, `AcraProxy`, `AcraConfigUI`, and key generators (`AcraGenKeys` and `AcraGenAuth`). You can find the containers in [/docker](https://github.com/cossacklabs/acra/tree/master/docker) folder or on the [Docker Hub](https://hub.docker.com/r/cossacklabs/) ([#139](https://github.com/cossacklabs/acra/pull/139)).
- Updated [Getting started with Docker](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker) guide to make starting out with Acra even easier.

- Added easy-to-use docker-compose files to launch Acra in different environments, including key distribution. Possible configurations are:
- `acraserver` + `acra_configui `;
- connecting to PostreSQL or MySQL databases;
- using Secure Session or SSL as transport encryption;
- with or without `acraproxy`;
- with or without zones.<br/>

This is huge! We encourage you to try it! Check out the instructions and examples in the [/docker](https://github.com/cossacklabs/acra/tree/master/docker) folder. ([#154](https://github.com/cossacklabs/acra/pull/154), [#146](https://github.com/cossacklabs/acra/pull/146), [#134](https://github.com/cossacklabs/acra/pull/134), [#133](https://github.com/cossacklabs/acra/pull/133), [#102](https://github.com/cossacklabs/acra/pull/102)).

- **Go versions**

- Updated the list of supported versions of Go. Every Acra component can now be built using Go >1.7, except `acra_rollback` that requires Go >1.8. No worries, you can still download Acra as a binary package anyway :)

- **OS**

- Dropped support of Debian Wheezy (no autotests, no precompiled binaries now).


_Documentation_:

- Updated [QuickStart](https://github.com/cossacklabs/acra/wiki/Quick-start-guide) documentation about launching and building Acra components.
- Described how to setup [AcraCensor](https://github.com/cossacklabs/acra/wiki/acracensor) and [AcraConfigUI](https://github.com/cossacklabs/acra/wiki/AcraConfigUI).
- Added more details and described new options (like using TLS and connecting to MySQL databases) for [AcraServer](https://github.com/cossacklabs/acra/wiki/How-AcraServer-works) and [AcraProxy](https://github.com/cossacklabs/acra/wiki/AcraProxy-and-AcraWriter).
- Described new [logging](https://github.com/cossacklabs/acra/wiki/Logging) formats.
- Updated description of [Key management](https://github.com/cossacklabs/acra/wiki/Key-Management) approach we encourage you to use.
- Described Docker components and ready-to-use Docker Compose configurations based on the [Docker Readme](https://github.com/cossacklabs/acra/tree/master/docker).
- Updated [Getting started with Docker](https://github.com/cossacklabs/acra/wiki/Trying-Acra-with-Docker) guide.
- Distributed the information about master key across the docs.
- Many small improvements.




## [0.76](https://github.com/cossacklabs/acra/releases/tag/0.76), March 9th 2018

Expand Down
98 changes: 73 additions & 25 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,22 +1,33 @@
ifneq ($(BUILD_PATH),)
BIN_PATH = $(BUILD_PATH)
BIN_PATH = $(BUILD_PATH)
else
BIN_PATH = build
BIN_PATH = build
endif
#default engine

ifeq ($(PREFIX),)
PREFIX = /usr
PREFIX = /usr
endif

TEMP_GOPATH = temp_gopath
ABS_TEMP_GOPATH = $(shell pwd)/$(TEMP_GOPATH)
ABS_TEMP_GOPATH := $(shell pwd)/$(TEMP_GOPATH)

ifneq ($(GIT_BRANCH),)
BRANCH = $(GIT_BRANCH)
else
BRANCH = master
endif

GIT_VERSION := $(shell if [ -d ".git" ]; then git version; fi 2>/dev/null)
ifdef GIT_VERSION
VERSION = $(shell git describe --tags HEAD | cut -b 1-)
VERSION = $(shell git describe --tags HEAD | cut -b 1-)
GIT_HASH = $(shell git rev-parse --verify HEAD)
else
VERSION = $(shell date -I)
VERSION = $(shell date -I)
endif

.PHONY: get_version dist temp_copy install clean test_go test_python test \
test_all unpack_dist deb rpm docker docker_push

get_version:
@echo $(VERSION)

Expand Down Expand Up @@ -59,7 +70,6 @@ test: temp_copy test_go
# alias for unification with other products
test_all: test


PACKAGE_NAME = acra
COSSACKLABS_URL = https://www.cossacklabs.com
MAINTAINER = "Cossack Labs Limited <dev@cossacklabs.com>"
Expand All @@ -69,36 +79,37 @@ LICENSE_NAME = "Apache License Version 2.0"

DEBIAN_CODENAME := $(shell lsb_release -cs 2> /dev/null)
DEBIAN_ARCHITECTURE = `dpkg --print-architecture 2>/dev/null`
DEBIAN_DEPENDENCIES := --depends openssl --depends libthemis
DEBIAN_DEPENDENCIES = --depends openssl --depends libthemis
RPM_DEPENDENCIES = --depends openssl --depends libthemis

ifeq ($(shell lsb_release -is 2> /dev/null),Debian)
NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb
OS_CODENAME = $(shell lsb_release -cs)
NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb
OS_CODENAME = $(shell lsb_release -cs)
else ifeq ($(shell lsb_release -is 2> /dev/null),Ubuntu)
NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb
OS_CODENAME = $(shell lsb_release -cs)
NAME_SUFFIX = $(VERSION)+$(DEBIAN_CODENAME)_$(DEBIAN_ARCHITECTURE).deb
OS_CODENAME = $(shell lsb_release -cs)
else
OS_NAME = $(shell cat /etc/os-release | grep -e "^ID=\".*\"" | cut -d'"' -f2)
OS_VERSION = $(shell cat /etc/os-release | grep -i version_id|cut -d'"' -f2)
ARCHITECTURE = $(shell arch)
RPM_VERSION = $(shell echo -n "$(VERSION)"|sed s/-/_/g)
NAME_SUFFIX = $(RPM_VERSION).$(OS_NAME)$(OS_VERSION).$(ARCHITECTURE).rpm
OS_NAME = $(shell cat /etc/os-release | grep -e "^ID=\".*\"" | cut -d'"' -f2)
OS_VERSION = $(shell cat /etc/os-release | grep -i version_id|cut -d'"' -f2)
ARCHITECTURE = $(shell arch)
RPM_VERSION = $(shell echo -n "$(VERSION)"|sed s/-/_/g)
NAME_SUFFIX = $(RPM_VERSION).$(OS_NAME)$(OS_VERSION).$(ARCHITECTURE).rpm
endif

SHORT_DESCRIPTION = "Acra helps you easily secure your databases in distributed, microservice-rich environments"
RPM_SUMMARY = "Acra helps you easily secure your databases in distributed, microservice-rich environments. \
It allows you to selectively encrypt sensitive records with strong multi-layer cryptography, detect potential \
intrusions and SQL injections and cryptographically compartmentalize data stored in large sharded schemes. \
Acra's security model guarantees that if your database or your application become compromised, they will not \
leak sensitive data, or keys to decrypt them."
It allows you to selectively encrypt sensitive records with strong multi-layer cryptography, detect potential \
intrusions and SQL injections and cryptographically compartmentalize data stored in large sharded schemes. \
Acra's security model guarantees that if your database or your application become compromised, they will not \
leak sensitive data, or keys to decrypt them."

BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')

unpack_dist:
@tar -xf $(DIST_FILENAME)

deb: install
@mkdir -p '$(BIN_PATH)/deb'

@fpm --input-type dir \
--output-type deb \
--name $(PACKAGE_NAME) \
Expand All @@ -113,11 +124,9 @@ deb: install
--deb-priority optional \
--category security \
$(TEMP_GOPATH)/bin/=$(PREFIX)/bin

# it's just for printing .deb files
@find $(BIN_PATH) -name \*.deb


rpm: install
@mkdir -p $(BIN_PATH)/rpm
@fpm --input-type dir \
Expand All @@ -135,3 +144,42 @@ rpm: install
$(TEMP_GOPATH)/bin/=$(PREFIX)/bin
# it's just for printing .rpm files
@find $(BIN_PATH) -name \*.rpm

define docker_build
@docker image build \
--no-cache=true \
--build-arg VERSION=$(VERSION)\
--build-arg VCS_URL="https://github.com/cossacklabs/acra" \
--build-arg VCS_REF=$(GIT_HASH) \
--build-arg VCS_BRANCH=$(BRANCH) \
--build-arg BUILD_DATE=$(BUILD_DATE) \
--tag cossacklabs/$(1):$(GIT_HASH) \
-f ./docker/$(1).dockerfile \
.
for tag in $(2); do \
docker tag cossacklabs/$(1):$(GIT_HASH) cossacklabs/$(1):$$tag; \
done
endef

ifeq ($(BRANCH),stable)
CONTAINER_TAGS = stable latest $(VERSION)
else ifeq ($(BRANCH),master)
CONTAINER_TAGS = master current $(VERSION)
endif

docker:
$(call docker_build,acra-build,)
$(call docker_build,acraserver,$(CONTAINER_TAGS))
$(call docker_build,acraproxy,$(CONTAINER_TAGS))
$(call docker_build,acra_genkeys,$(CONTAINER_TAGS))
$(call docker_build,acra_configui,$(CONTAINER_TAGS))
$(call docker_build,acra_genauth,$(CONTAINER_TAGS))
@docker image rm cossacklabs/acra-build:$(GIT_HASH)

docker_push: docker
@docker push cossacklabs/acraserver
@docker push cossacklabs/acraproxy
@docker push cossacklabs/acra_genkeys
@docker push cossacklabs/acra_genkeys
@docker push cossacklabs/acra_configui
@docker push cossacklabs/acra_genauth
Loading

0 comments on commit d619a20

Please sign in to comment.