From 39ba6927dd01e18c0fda557e0d54ed325e1e27ba Mon Sep 17 00:00:00 2001 From: Gabriel Saratura Date: Fri, 23 Sep 2022 13:13:52 +0300 Subject: [PATCH 1/3] Add database support --- go.mod | 15 +- go.sum | 124 ++++++++++++++++- main.go | 24 +--- pkg/database/database.go | 251 ++++++++++++++++++++++++++++++++++ pkg/exoscale/exoscale.go | 19 +++ pkg/kubernetes/kubernetes.go | 26 ++++ pkg/sos/objectstorage.go | 196 ++++++++++++++++++++++++++ pkg/sos/objectstorage_test.go | 0 sos_command.go | 84 ++++++++---- src/exoscale/exoscale.go | 38 ----- src/kubernetes/kubernetes.go | 65 --------- src/sos/objectstorage.go | 122 ----------------- 12 files changed, 693 insertions(+), 271 deletions(-) create mode 100644 pkg/database/database.go create mode 100644 pkg/exoscale/exoscale.go create mode 100644 pkg/kubernetes/kubernetes.go create mode 100644 pkg/sos/objectstorage.go create mode 100644 pkg/sos/objectstorage_test.go delete mode 100644 src/exoscale/exoscale.go delete mode 100644 src/kubernetes/kubernetes.go delete mode 100644 src/sos/objectstorage.go diff --git a/go.mod b/go.mod index 1b29858..78abee6 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,14 @@ module github.com/vshn/exoscale-metrics-collector go 1.18 require ( + github.com/appuio/appuio-cloud-reporting v0.5.0 github.com/ccremer/go-command-pipeline v0.20.0 github.com/exoscale/egoscale v0.90.1 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 - github.com/urfave/cli/v2 v2.16.3 + github.com/jmoiron/sqlx v1.3.5 + github.com/urfave/cli/v2 v2.17.1 + github.com/vshn/cloudscale-metrics-collector v0.3.3 github.com/vshn/provider-exoscale v0.1.0 go.uber.org/zap v1.23.0 golang.org/x/exp v0.0.0-20220930202632-ec3f01382ef9 @@ -39,8 +42,17 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.12.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.11.0 // indirect + github.com/jackc/pgx/v4 v4.16.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lopezator/migrator v0.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -60,6 +72,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect diff --git a/go.sum b/go.sum index a4cfaf7..a12b0ea 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -45,6 +47,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/appuio/appuio-cloud-reporting v0.5.0 h1:u/EaddJ20FCU9sAW/QLgo2KUBGC2qBTckzyyBTvlYBM= +github.com/appuio/appuio-cloud-reporting v0.5.0/go.mod h1:/S0E32wNNoF8Cy1bdY9ewXqvNRFA6u+BSJ61+5SF+ek= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -67,8 +71,13 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crossplane/crossplane-runtime v0.17.0 h1:gt2JcOYcVBw/luQToq2hUkoersL12ICuV0YzKI5lyCs= github.com/crossplane/crossplane-runtime v0.17.0/go.mod h1:IPT3HTsovwmbw3i+SdsOyaC3r3b7TW+otBMmZsHLnSU= @@ -138,9 +147,13 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.7.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -230,7 +243,56 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0= +github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk= +github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -247,6 +309,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -256,6 +319,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -270,17 +334,30 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lopezator/migrator v0.3.0 h1:VW/rR+J8NYwPdkBxjrFdjwejpgvP59LbmANJxXuNbuk= +github.com/lopezator/migrator v0.3.0/go.mod h1:bpVAVPkWSvTw8ya2Pk7E/KiNAyDWNImgivQY79o8/8I= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -337,9 +414,17 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -351,6 +436,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -366,10 +452,12 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= -github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk= -github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/urfave/cli/v2 v2.17.1 h1:UzjDEw2dJQUE3iRaiNQ1VrVFbyAtKGH3VdkMoHA58V0= +github.com/urfave/cli/v2 v2.17.1/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vshn/cloudscale-metrics-collector v0.3.3 h1:81y66v4WV1dPYYf+gpPwyxHa2bDb+igSXXUIZF/1vrg= +github.com/vshn/cloudscale-metrics-collector v0.3.3/go.mod h1:Ejyqp4eZDBrjnXJPA66ttA8cI3I06aa6W2CGxV2DB2o= github.com/vshn/provider-exoscale v0.1.0 h1:2qEYt37BYvTkIvPUYWmZ2pZ7stAy4rGl4CvqTL86CTk= github.com/vshn/provider-exoscale v0.1.0/go.mod h1:ilQ+p905LOQByfrwPMnm+kfYRoRn8yXax4i1TJaNYhQ= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -381,6 +469,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -388,28 +477,45 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -459,6 +565,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -513,7 +620,9 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -521,6 +630,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -594,14 +704,18 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -610,6 +724,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -639,6 +754,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -752,6 +869,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/main.go b/main.go index 70a8884..3c2d69d 100644 --- a/main.go +++ b/main.go @@ -54,12 +54,14 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) { Before: setupLogging, Flags: []cli.Flag{ &cli.IntFlag{ - Name: "log-level", Aliases: []string{"v"}, EnvVars: envVars("LOG_LEVEL"), - Usage: "number of the log level verbosity", - Value: 0, + Name: "log-level", Aliases: []string{"v"}, + EnvVars: []string{"LOG_LEVEL"}, + Usage: "number of the log level verbosity", + Value: 0, }, &cli.StringFlag{ - Name: "log-format", EnvVars: envVars("LOG_FORMAT"), + Name: "log-format", + EnvVars: []string{"LOG_FORMAT"}, Usage: "sets the log format (values: [json, console])", DefaultText: "console", }, @@ -90,17 +92,3 @@ func rootAction(hasSubcommands bool) func(context *cli.Context) error { return LogMetadata(ctx) } } - -// env combines envPrefix with given suffix delimited by underscore. -func env(suffix string) string { - return envPrefix + suffix -} - -// envVars combines envPrefix with each given suffix delimited by underscore. -func envVars(suffixes ...string) []string { - arr := make([]string, len(suffixes)) - for i := range suffixes { - arr[i] = env(suffixes[i]) - } - return arr -} diff --git a/pkg/database/database.go b/pkg/database/database.go new file mode 100644 index 0000000..7ce04ac --- /dev/null +++ b/pkg/database/database.go @@ -0,0 +1,251 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "github.com/appuio/appuio-cloud-reporting/pkg/db" + pipeline "github.com/ccremer/go-command-pipeline" + "github.com/jmoiron/sqlx" + "github.com/vshn/cloudscale-metrics-collector/pkg/categoriesmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/datetimesmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/discountsmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/factsmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/productsmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/queriesmodel" + "github.com/vshn/cloudscale-metrics-collector/pkg/tenantsmodel" + "strings" + "time" + + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + sourceQueryStorage = "object-storage-storage" + provider = "exoscale" + queryAndZone = sourceQueryStorage + ":" + provider + defaultUnit = "GBDay" +) + +var ( + product = db.Product{ + Source: queryAndZone, + Target: sql.NullString{String: "1402", Valid: true}, + Amount: 0.00066, + Unit: "GBDay", + During: db.InfiniteRange(), + } + discount = db.Discount{ + Source: sourceQueryStorage, + Discount: 0, + During: db.InfiniteRange(), + } + query = db.Query{ + Name: queryAndZone, + Description: "Object Storage - Storage (exoscale.com)", + Query: "", + Unit: "GBDay", + During: db.InfiniteRange(), + } +) + +// AggregatedBucket contains total used storage in an organization namespace +type AggregatedBucket struct { + Organization string + // Storage in bytes + StorageUsed float64 +} + +// Database holds raw url of the postgresql database with the opened connection +type Database struct { + URL string + BillingDate time.Time + connection *sqlx.DB +} + +type transactionContext struct { + context.Context + billingDate time.Time + namespace *string + aggregatedBucket *AggregatedBucket + transaction *sqlx.Tx + tenant *db.Tenant + category *db.Category + dateTime *db.DateTime + product *db.Product + discount *db.Discount + query *db.Query + quantity *float64 +} + +// OpenConnection opens a connection to the postgres database +func (d *Database) OpenConnection() error { + connection, err := db.Openx(d.URL) + if err != nil { + return fmt.Errorf("cannot create a connection to the database: %w", err) + } + d.connection = connection + return nil +} + +// CloseConnection closes the connection to the postgres database +func (d *Database) CloseConnection() error { + err := d.connection.Close() + if err != nil { + return fmt.Errorf("cannot close database connection: %w", err) + } + return nil +} + +// EnsureBucketUsage saves the aggregated buckets usage by namespace to the postgresql database +// To save the correct data to the database the function also matches a relevant product, discount (if any) and query. +// The storage usage is referred to a day before the application ran (yesterday) +func (d *Database) EnsureBucketUsage(ctx context.Context, namespace string, aggregatedBucket AggregatedBucket) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Saving buckets usage for namespace", "namespace", namespace, "storage used", aggregatedBucket.StorageUsed) + + tctx := &transactionContext{ + Context: ctx, + namespace: &namespace, + aggregatedBucket: &aggregatedBucket, + billingDate: d.BillingDate, + } + p := pipeline.NewPipeline[*transactionContext]() + p.WithSteps( + p.NewStep("Begin database transaction", d.beginTransaction), + p.NewStep("Ensure necessary models", d.ensureModels), + p.NewStep("Get best match", d.getBestMatch), + p.NewStep("Adjust storage size", d.adjustStorageSizeUnit), + p.NewStep("Save facts", d.saveFacts), + p.NewStep("Commit transaction", d.commitTransaction), + ) + err := p.RunWithContext(tctx) + if err != nil { + log.Info("Buckets usage have not been saved to the database", "namespace", namespace, "error", err.Error()) + tctx.transaction.Rollback() + return err + } + return nil +} + +func (d *Database) beginTransaction(ctx *transactionContext) error { + tx, err := d.connection.BeginTxx(ctx, &sql.TxOptions{}) + if err != nil { + return fmt.Errorf("cannot create database transaction for namespace %s: %w", *ctx.namespace, err) + } + ctx.transaction = tx + return nil +} + +func (d *Database) ensureModels(ctx *transactionContext) error { + namespace := *ctx.namespace + tenant, err := tenantsmodel.Ensure(ctx, ctx.transaction, &db.Tenant{Source: ctx.aggregatedBucket.Organization}) + if err != nil { + return fmt.Errorf("cannot ensure organization for namespace %s: %w", namespace, err) + } + ctx.tenant = tenant + + category, err := categoriesmodel.Ensure(ctx, ctx.transaction, &db.Category{Source: provider + ":" + namespace}) + if err != nil { + return fmt.Errorf("cannot ensure category for namespace %s: %w", namespace, err) + } + ctx.category = category + + dateTime := datetimesmodel.New(ctx.billingDate) + dateTime, err = datetimesmodel.Ensure(ctx, ctx.transaction, dateTime) + if err != nil { + return fmt.Errorf("cannot ensure date time for namespace %s: %w", namespace, err) + } + ctx.dateTime = dateTime + return nil +} + +func (d *Database) getBestMatch(ctx *transactionContext) error { + namespace := *ctx.namespace + productMatch, err := productsmodel.GetBestMatch(ctx, ctx.transaction, getSourceString(namespace, ctx.aggregatedBucket.Organization), ctx.billingDate) + if err != nil { + return fmt.Errorf("cannot get product best match for namespace %s: %w", namespace, err) + } + ctx.product = productMatch + + discountMatch, err := discountsmodel.GetBestMatch(ctx, ctx.transaction, getSourceString(namespace, ctx.aggregatedBucket.Organization), ctx.billingDate) + if err != nil { + return fmt.Errorf("cannot get discount best match for namespace %s: %w", namespace, err) + } + ctx.discount = discountMatch + + queryMatch, err := queriesmodel.GetByName(ctx, ctx.transaction, queryAndZone) + if err != nil { + return fmt.Errorf("cannot get query by name for namespace %s: %w", namespace, err) + } + ctx.query = queryMatch + + return nil +} + +func (d *Database) adjustStorageSizeUnit(ctx *transactionContext) error { + var quantity float64 + if query.Unit == defaultUnit { + quantity = ctx.aggregatedBucket.StorageUsed / 1024 / 1024 / 1024 + } else { + return fmt.Errorf("unknown query unit %s", query.Unit) + } + ctx.quantity = &quantity + return nil +} + +func (d *Database) saveFacts(ctx *transactionContext) error { + storageFact := factsmodel.New(ctx.dateTime, ctx.query, ctx.tenant, ctx.category, ctx.product, ctx.discount, *ctx.quantity) + _, err := factsmodel.Ensure(ctx, ctx.transaction, storageFact) + if err != nil { + return fmt.Errorf("cannot save fact for namespace %s: %w", *ctx.namespace, err) + } + return nil +} + +func (d *Database) commitTransaction(ctx *transactionContext) error { + err := ctx.transaction.Commit() + if err != nil { + return fmt.Errorf("cannot commit transaction for buckets in namespace %s: %w", *ctx.namespace, err) + } + return nil +} + +// EnsureInitConfiguration ensures the minimum exoscale object storage configuration data is present in the database +// before saving buckets usage +func (d *Database) EnsureInitConfiguration(ctx context.Context) error { + transaction, err := d.connection.BeginTxx(ctx, &sql.TxOptions{}) + if err != nil { + return fmt.Errorf("cannot begin transaction for initial database configuration: %w", err) + } + defer transaction.Rollback() + err = ensureInitConfigurationModels(ctx, err, transaction) + if err != nil { + return err + } + err = transaction.Commit() + if err != nil { + return fmt.Errorf("cannot commit transaction for initial database configuration: %w", err) + } + return nil +} + +func ensureInitConfigurationModels(ctx context.Context, err error, transaction *sqlx.Tx) error { + _, err = productsmodel.Ensure(ctx, transaction, &product) + if err != nil { + return fmt.Errorf("cannot ensure exoscale product model in the database: %w", err) + } + _, err = discountsmodel.Ensure(ctx, transaction, &discount) + if err != nil { + return fmt.Errorf("cannot ensure exoscale discount model in the database: %w", err) + } + _, err = queriesmodel.Ensure(ctx, transaction, &query) + if err != nil { + return fmt.Errorf("cannot ensure exoscale query model in the database: %w", err) + } + return nil +} + +func getSourceString(namespace, organization string) string { + return strings.Join([]string{queryAndZone, organization, namespace}, ":") +} diff --git a/pkg/exoscale/exoscale.go b/pkg/exoscale/exoscale.go new file mode 100644 index 0000000..c88ea34 --- /dev/null +++ b/pkg/exoscale/exoscale.go @@ -0,0 +1,19 @@ +package exoscale + +import ( + "fmt" + egoscale "github.com/exoscale/egoscale/v2" +) + +// sosEndpoint has buckets across all zones +const sosEndpoint = "https://api-ch-gva-2.exoscale.com" + +// InitClient creates exoscale client with given access and secret keys +func InitClient(exoscaleAccessKey, exoscaleSecret string) (*egoscale.Client, error) { + options := egoscale.ClientOptWithAPIEndpoint(sosEndpoint) + client, err := egoscale.NewClient(exoscaleAccessKey, exoscaleSecret, options) + if err != nil { + return nil, fmt.Errorf("cannot create Exoscale client: %w", err) + } + return client, nil +} diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go new file mode 100644 index 0000000..a7e2bfb --- /dev/null +++ b/pkg/kubernetes/kubernetes.go @@ -0,0 +1,26 @@ +package kubernetes + +import ( + "fmt" + "github.com/vshn/provider-exoscale/apis" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InitK8sClient creates a k8s client from the server url and token url +func InitK8sClient(url, token string) (*client.Client, error) { + scheme := runtime.NewScheme() + err := apis.AddToScheme(scheme) + if err != nil { + return nil, fmt.Errorf("cannot add k8s exoscale scheme: %w", err) + } + config := rest.Config{Host: url, BearerToken: token} + k8sClient, e := client.New(&config, client.Options{ + Scheme: scheme, + }) + if e != nil { + return nil, fmt.Errorf("cannot initialise k8s client: %w", err) + } + return &k8sClient, nil +} diff --git a/pkg/sos/objectstorage.go b/pkg/sos/objectstorage.go new file mode 100644 index 0000000..d320221 --- /dev/null +++ b/pkg/sos/objectstorage.go @@ -0,0 +1,196 @@ +package sos + +import ( + "context" + "fmt" + "github.com/exoscale/egoscale/v2/oapi" + "time" + + pipeline "github.com/ccremer/go-command-pipeline" + egoscale "github.com/exoscale/egoscale/v2" + db "github.com/vshn/exoscale-metrics-collector/pkg/database" + exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" + ctrl "sigs.k8s.io/controller-runtime" + k8s "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + organizationLabel = "appuio.io/organization" + namespaceLabel = "crossplane.io/claim-namespace" + // Exoscale gathers metrics of its bucket at 6AM UTC + exoscaleTimeZone = "UTC" + exoscaleBillingHour = 6 +) + +// ObjectStorage gathers bucket data from exoscale provider and cluster and saves to the database +type ObjectStorage struct { + k8sClient k8s.Client + exoscaleClient *egoscale.Client + database *db.Database + bucketDetails []BucketDetail + aggregatedBuckets map[string]db.AggregatedBucket +} + +// BucketDetail a k8s bucket object with relevant data +type BucketDetail struct { + Organization, BucketName, Namespace string +} + +//NewObjectStorage creates an ObjectStorage with the initial setup +func NewObjectStorage(exoscaleClient *egoscale.Client, k8sClient *k8s.Client, databaseURL string) ObjectStorage { + return ObjectStorage{ + exoscaleClient: exoscaleClient, + k8sClient: *k8sClient, + database: &db.Database{URL: databaseURL}, + } +} + +// Execute executes the main business logic for this application by gathering, matching and saving data to the database +func (o *ObjectStorage) Execute(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Running metrics collector by step") + + p := pipeline.NewPipeline[context.Context]() + p.WithSteps( + p.NewStep("Fetch managed buckets", o.fetchManagedBuckets), + p.NewStep("Get bucket usage", o.getBucketUsage), + p.NewStep("Get billing date", o.getBillingDate), + p.NewStep("Save to database", o.saveToDatabase), + ) + return p.RunWithContext(ctx) +} + +// getBucketUsage gets bucket usage from Exoscale and matches them with the bucket from the cluster +// If there are no buckets in Exoscale, the API will return an empty slice +func (o *ObjectStorage) getBucketUsage(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Fetching bucket usage from Exoscale") + resp, err := o.exoscaleClient.ListSosBucketsUsageWithResponse(ctx) + if err != nil { + return err + } + + aggregatedBuckets := getAggregatedBuckets(ctx, *resp.JSON200.SosBucketsUsage, o.bucketDetails) + if len(aggregatedBuckets) == 0 { + log.Info("There are no bucket usage to be saved in the database") + return nil + } + + o.aggregatedBuckets = aggregatedBuckets + return nil +} + +func (o *ObjectStorage) fetchManagedBuckets(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Fetching buckets from cluster") + + buckets := exoscalev1.BucketList{} + log.V(1).Info("Listing buckets from cluster") + err := o.k8sClient.List(ctx, &buckets) + if err != nil { + return fmt.Errorf("cannot list buckets: %w", err) + } + o.bucketDetails = addOrgAndNamespaceToBucket(ctx, buckets) + return nil +} + +func (o *ObjectStorage) saveToDatabase(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Creating a database connection") + + log.V(1).Info("Opening database connection") + err := o.database.OpenConnection() + if err != nil { + return err + } + defer o.database.CloseConnection() + + log.V(1).Info("Ensuring initial database configuration") + err = o.database.EnsureInitConfiguration(ctx) + if err != nil { + return err + } + + log.V(1).Info("Saving buckets information usage to database") + for namespace, aggregatedBucket := range o.aggregatedBuckets { + err = o.database.EnsureBucketUsage(ctx, namespace, aggregatedBucket) + if err != nil { + return err + } + } + + return nil +} + +func (o *ObjectStorage) getBillingDate(_ context.Context) error { + location, err := time.LoadLocation(exoscaleTimeZone) + if err != nil { + return fmt.Errorf("cannot initialize location from time zone %s: %w", location, err) + } + now := time.Now().In(location) + previousDay := now.Day() - 1 + o.database.BillingDate = time.Date(now.Year(), now.Month(), previousDay, exoscaleBillingHour, 0, 0, 0, now.Location()) + return nil +} + +func getAggregatedBuckets(ctx context.Context, sosBucketsUsage []oapi.SosBucketUsage, bucketDetails []BucketDetail) map[string]db.AggregatedBucket { + log := ctrl.LoggerFrom(ctx) + log.Info("Aggregating buckets by namespace") + + sosBucketsUsageMap := make(map[string]oapi.SosBucketUsage, len(sosBucketsUsage)) + for _, usage := range sosBucketsUsage { + sosBucketsUsageMap[*usage.Name] = usage + } + + aggregatedBuckets := make(map[string]db.AggregatedBucket) + for _, bucketDetail := range bucketDetails { + log.V(1).Info("Checking bucket", "bucket", bucketDetail.BucketName) + + if bucketUsage, exists := sosBucketsUsageMap[bucketDetail.BucketName]; exists { + log.V(1).Info("Found exoscale bucket usage", "bucket", bucketUsage.Name, "bucket size", bucketUsage.Name) + aggregatedBucket := aggregatedBuckets[bucketDetail.Namespace] + aggregatedBucket.Organization = bucketDetail.Organization + aggregatedBucket.StorageUsed += float64(*bucketUsage.Size) + aggregatedBuckets[bucketDetail.Namespace] = aggregatedBucket + } else { + log.Info("Could not find any bucket on exoscale", "bucket", bucketDetail.BucketName) + } + } + return aggregatedBuckets +} + +func addOrgAndNamespaceToBucket(ctx context.Context, buckets exoscalev1.BucketList) []BucketDetail { + log := ctrl.LoggerFrom(ctx) + log.V(1).Info("Gathering more information from buckets") + + bucketDetails := make([]BucketDetail, 0, 10) + for _, bucket := range buckets.Items { + bucketDetail := BucketDetail{ + BucketName: bucket.Spec.ForProvider.BucketName, + } + if organization, exist := bucket.ObjectMeta.Labels[organizationLabel]; exist { + bucketDetail.Organization = organization + } else { + // cannot get organization from bucket + log.Info("Organization label is missing in bucket, skipping...", + "label", organizationLabel, + "bucket", bucket.Name) + continue + } + if namespace, exist := bucket.ObjectMeta.Labels[namespaceLabel]; exist { + bucketDetail.Namespace = namespace + } else { + // cannot get namespace from bucket + log.Info("Namespace label is missing in bucket, skipping...", + "label", namespaceLabel, + "bucket", bucket.Name) + continue + } + log.V(1).Info("Added namespace and organization to bucket", + "bucket", bucket.Name, + "namespace", bucketDetail.Namespace, + "organization", bucketDetail.Organization) + bucketDetails = append(bucketDetails, bucketDetail) + } + return bucketDetails +} diff --git a/pkg/sos/objectstorage_test.go b/pkg/sos/objectstorage_test.go new file mode 100644 index 0000000..e69de29 diff --git a/sos_command.go b/sos_command.go index 1ea8ea9..9c4cbae 100644 --- a/sos_command.go +++ b/sos_command.go @@ -2,18 +2,27 @@ package main import ( "github.com/urfave/cli/v2" - "github.com/vshn/exoscale-metrics-collector/src/exoscale" - "github.com/vshn/exoscale-metrics-collector/src/kubernetes" - "github.com/vshn/exoscale-metrics-collector/src/sos" + "github.com/vshn/exoscale-metrics-collector/pkg/exoscale" + k8s "github.com/vshn/exoscale-metrics-collector/pkg/kubernetes" + "github.com/vshn/exoscale-metrics-collector/pkg/sos" ctrl "sigs.k8s.io/controller-runtime" ) const ( - objectStorageName = "objectstorage" + objectStorageName = "objectstorage" + keyEnvVariable = "EXOSCALE_API_KEY" + secretEnvVariable = "EXOSCALE_API_SECRET" + dbURLEnvVariable = "ACR_DB_URL" + k8sServerURLEnvVariable = "K8S_SERVER_URL" + k8sTokenEnvVariable = "K8S_TOKEN" ) type objectStorageCommand struct { - clusterFilePath string + clusterURL string + clusterToken string + databaseURL string + exoscaleKey string + exoscaleSecret string } func NewCommand() *cli.Command { @@ -23,9 +32,45 @@ func NewCommand() *cli.Command { Usage: "Get metrics from object storage service", Action: command.execute, Flags: []cli.Flag{ - &cli.StringFlag{Name: "path-to-cluster-file", Aliases: []string{"clusters"}, Required: true, TakesFile: true, - Usage: "File containing a list of clusters with the format server=token..", - Destination: &command.clusterFilePath, + &cli.StringFlag{ + Name: "k8s-server-url", + Aliases: []string{"u"}, + EnvVars: []string{k8sServerURLEnvVariable}, + Required: true, + Usage: "A Kubernetes server URL from where to get the data from", + Destination: &command.clusterURL, + }, + &cli.StringFlag{ + Name: "k8s-server-token", + Aliases: []string{"t"}, + EnvVars: []string{k8sTokenEnvVariable}, + Required: true, + Usage: "A Kubernetes server token which can view buckets.exoscale.crossplane.io resources", + Destination: &command.clusterToken, + }, + &cli.StringFlag{ + Name: "database-url", + Aliases: []string{"d"}, + EnvVars: []string{dbURLEnvVariable}, + Required: true, + Usage: "A PostgreSQL database URL where to save relevant metrics", + Destination: &command.databaseURL, + }, + &cli.StringFlag{ + Name: "exoscale-access-key", + Aliases: []string{"k"}, + EnvVars: []string{keyEnvVariable}, + Required: true, + Usage: "A key which has unrestricted SOS service access in an Exoscale organization", + Destination: &command.exoscaleKey, + }, + &cli.StringFlag{ + Name: "exoscale-secret", + Aliases: []string{"s"}, + EnvVars: []string{secretEnvVariable}, + Required: true, + Usage: "The secret which has unrestricted SOS service access in an Exoscale organization", + Destination: &command.exoscaleSecret, }, }, } @@ -34,28 +79,19 @@ func NewCommand() *cli.Command { func (c *objectStorageCommand) execute(ctx *cli.Context) error { log := AppLogger(ctx).WithName(objectStorageName) ctrl.SetLogger(log) - log.Info("Loading Exoscale API credentials") - accessKey, secretKey, err := exoscale.LoadAPICredentials() - if err != nil { - return err - } + log.Info("Creating Exoscale client") - exoscaleClient, err := exoscale.CreateExoscaleClient(accessKey, secretKey) - if err != nil { - return err - } - log.Info("Reading input cluster file configuration") - clusters, err := kubernetes.ReadConf(c.clusterFilePath) + exoscaleClient, err := exoscale.InitClient(c.exoscaleKey, c.exoscaleSecret) if err != nil { return err } - log.Info("Creating k8s clients") - err = kubernetes.InitKubernetesClients(ctx.Context, clusters) + + log.Info("Creating k8s client") + k8sClient, err := k8s.InitK8sClient(c.clusterURL, c.clusterToken) if err != nil { return err } - o := sos.NewObjectStorage(exoscaleClient, clusters) - err = o.Execute(ctx.Context) - return err + o := sos.NewObjectStorage(exoscaleClient, k8sClient, c.databaseURL) + return o.Execute(ctx.Context) } diff --git a/src/exoscale/exoscale.go b/src/exoscale/exoscale.go deleted file mode 100644 index 39e03e6..0000000 --- a/src/exoscale/exoscale.go +++ /dev/null @@ -1,38 +0,0 @@ -package exoscale - -import ( - "fmt" - egoscale "github.com/exoscale/egoscale/v2" - "os" -) - -const ( - keyEnvVariable = "EXOSCALE_API_KEY" - secretEnvVariable = "EXOSCALE_API_SECRET" - - // sosEndpoint has buckets across all zones - sosEndpoint = "https://api-ch-gva-2.exoscale.com" -) - -// LoadAPICredentials retrieves exoscale access and secret keys from environment variables -func LoadAPICredentials() (exoscaleAccessKey, exoscaleSecretKey string, err error) { - APIKey := os.Getenv(keyEnvVariable) - if APIKey == "" { - return "", "", fmt.Errorf("cannot find environment variable %s", keyEnvVariable) - } - APISecret := os.Getenv(secretEnvVariable) - if APISecret == "" { - return "", "", fmt.Errorf("cannot find environment variable %s", secretEnvVariable) - } - return APIKey, APISecret, nil -} - -// CreateExoscaleClient creates exoscale client with given access and secret keys -func CreateExoscaleClient(exoscaleAccessKey, exoscaleSecretKey string) (*egoscale.Client, error) { - options := egoscale.ClientOptWithAPIEndpoint(sosEndpoint) - client, err := egoscale.NewClient(exoscaleAccessKey, exoscaleSecretKey, options) - if err != nil { - return nil, fmt.Errorf("cannot create Exoscale client: %w", err) - } - return client, err -} diff --git a/src/kubernetes/kubernetes.go b/src/kubernetes/kubernetes.go deleted file mode 100644 index 9713fe3..0000000 --- a/src/kubernetes/kubernetes.go +++ /dev/null @@ -1,65 +0,0 @@ -package kubernetes - -import ( - "context" - "fmt" - "github.com/vshn/provider-exoscale/apis" - "io/ioutil" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type Cluster struct { - client.Client - Name string - Url string - Token string -} - -// ReadConf reads a yaml config file and unmarshalls it into an object of type Cluster -func ReadConf(filename string) (*[]Cluster, error) { - buf, err := ioutil.ReadFile(filename) - if err != nil { - return nil, fmt.Errorf("cannot read cluster config file %q: %w", filename, err) - } - - var clusters []Cluster - err = yaml.Unmarshal(buf, &clusters) - if err != nil { - return nil, fmt.Errorf("cannot unmarshall cluster config file %q: %w", filename, err) - } - - return &clusters, err -} - -// InitKubernetesClients creates as many k8s clients as there are cluster entries from the parsed file -// The function skips clients that cannot be initialized -func InitKubernetesClients(ctx context.Context, clusters *[]Cluster) error { - log := ctrl.LoggerFrom(ctx) - scheme := runtime.NewScheme() - err := apis.AddToScheme(scheme) - if err != nil { - return fmt.Errorf("cannot add k8s exoscale scheme: %w", err) - } - noAvailableClients := true - for index, cluster := range *clusters { - config := rest.Config{Host: cluster.Url, BearerToken: cluster.Token} - clientInstance, e := client.New(&config, client.Options{ - Scheme: scheme, - }) - (*clusters)[index].Client = clientInstance - if e != nil { - // Continue in case a k8s client cannot be initiated. - log.Info("Cannot initiate k8s client, skipping config", "cluster", cluster.Url, "error", e.Error()) - continue - } - noAvailableClients = false - } - if noAvailableClients { - return fmt.Errorf("there are no valid k8s clients configured") - } - return nil -} diff --git a/src/sos/objectstorage.go b/src/sos/objectstorage.go deleted file mode 100644 index fb5c709..0000000 --- a/src/sos/objectstorage.go +++ /dev/null @@ -1,122 +0,0 @@ -package sos - -import ( - "context" - "fmt" - pipeline "github.com/ccremer/go-command-pipeline" - egoscale "github.com/exoscale/egoscale/v2" - "github.com/vshn/exoscale-metrics-collector/src/kubernetes" - exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" - "golang.org/x/exp/slices" - ctrl "sigs.k8s.io/controller-runtime" -) - -var ( - organizationLabel = "appuio.io/organization" - namespaceLabel = "crossplane.io/claim-namespace" -) - -type ObjectStorage struct { - k8sClusters *[]kubernetes.Cluster - exoscaleClient *egoscale.Client - bucketDetails []BucketDetail -} - -type BucketDetail struct { - ClusterName string - Namespace string - Organization string - BucketName string -} - -func NewObjectStorage(exoscaleClient *egoscale.Client, k8sClusters *[]kubernetes.Cluster) ObjectStorage { - return ObjectStorage{ - exoscaleClient: exoscaleClient, - k8sClusters: k8sClusters, - } -} - -func (o *ObjectStorage) Execute(ctx context.Context) error { - log := ctrl.LoggerFrom(ctx) - log.Info("Running metrics collector by step") - - p := pipeline.NewPipeline[context.Context]() - p.WithSteps( - p.NewStep("Fetch managed buckets", o.fetchManagedBuckets), - p.NewStep("Get bucket usage", o.getBucketUsage), - ) - return p.RunWithContext(ctx) -} - -func (o *ObjectStorage) getBucketUsage(ctx context.Context) error { - log := ctrl.LoggerFrom(ctx) - log.Info("Fetching bucket usage from Exoscale") - resp, err := o.exoscaleClient.ListSosBucketsUsageWithResponse(ctx) - if err != nil { - return err - } - - for _, bucketUsage := range *resp.JSON200.SosBucketsUsage { - name := *bucketUsage.Name - zoneName := *bucketUsage.ZoneName - size := *bucketUsage.Size - createdAt := *bucketUsage.CreatedAt - - log.V(1).Info("Trying to find a match in a cluster", "bucket name", name) - // Match bucket name from exoscale with the bucket found in one of the clusters - bucketIndex := slices.IndexFunc(o.bucketDetails, func(ni BucketDetail) bool { return ni.BucketName == name }) - - if bucketIndex == -1 { - log.Info("Cannot find bucket in any cluster", "bucket name", name) - continue - } - fmt.Printf("name: %s, zoneName: %s, size: %d, createdAt: %s, namespace info: %s\n", name, zoneName, size, createdAt, o.bucketDetails[bucketIndex]) - } - return nil -} - -func (o *ObjectStorage) fetchManagedBuckets(ctx context.Context) error { - log := ctrl.LoggerFrom(ctx) - log.Info("Fetching buckets from clusters") - for _, k8sCluster := range *o.k8sClusters { - buckets := exoscalev1.BucketList{} - log.V(1).Info("Listing buckets from cluster", "cluster name", k8sCluster.Name) - err := k8sCluster.List(ctx, &buckets) - if err != nil { - return fmt.Errorf("cannot list buckets: %w", err) - } - bucketDetailsByCluster := matchOrgAndNamespaceByBucket(ctx, &buckets, k8sCluster.Name) - - // add BucketDetail from each cluster to one single slice - o.bucketDetails = append(o.bucketDetails, *bucketDetailsByCluster...) - } - return nil -} - -func matchOrgAndNamespaceByBucket(ctx context.Context, buckets *exoscalev1.BucketList, clusterName string) *[]BucketDetail { - log := ctrl.LoggerFrom(ctx) - log.V(1).Info("Gathering more information from buckets", "cluster name", clusterName) - - var bucketDetails []BucketDetail - for _, bucket := range buckets.Items { - log.V(1).Info("Gathering more information on bucket", "bucket name", bucket.Name, "cluster name", clusterName) - bucketDetail := BucketDetail{ - BucketName: bucket.Spec.ForProvider.BucketName, - ClusterName: clusterName, - } - if organization, exist := bucket.ObjectMeta.Labels[organizationLabel]; exist { - bucketDetail.Organization = organization - } else { - // cannot get organization from bucket - log.Info("Organization label is missing in bucket", "bucket name", bucket.Name, "label", organizationLabel) - } - if namespace, exist := bucket.ObjectMeta.Labels[namespaceLabel]; exist { - bucketDetail.Namespace = namespace - } else { - // cannot get namespace from bucket - log.Info("Namespace label is missing in bucket", "bucket name", bucket.Name, "label", namespaceLabel) - } - bucketDetails = append(bucketDetails, bucketDetail) - } - return &bucketDetails -} From 3a61cf94a4597ea57a4e4d897e783f547696c4db Mon Sep 17 00:00:00 2001 From: Gabriel Saratura Date: Wed, 5 Oct 2022 09:56:19 +0300 Subject: [PATCH 2/3] Add unit tests --- pkg/sos/objectstorage_test.go | 199 ++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/pkg/sos/objectstorage_test.go b/pkg/sos/objectstorage_test.go index e69de29..4380977 100644 --- a/pkg/sos/objectstorage_test.go +++ b/pkg/sos/objectstorage_test.go @@ -0,0 +1,199 @@ +package sos + +import ( + "context" + "github.com/exoscale/egoscale/v2/oapi" + "github.com/stretchr/testify/assert" + db "github.com/vshn/exoscale-metrics-collector/pkg/database" + exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "time" +) + +func TestObjectStorage_GetBillingDate(t *testing.T) { + t.Run("GivenContext_WhenGetBillingDate_ThenReturnYesterdayDate", func(t *testing.T) { + // Given + ctx := context.Background() + utc, _ := time.LoadLocation("UTC") + now := time.Now().In(utc) + expected := time.Date(now.Year(), now.Month(), time.Now().Day()-1, 6, 0, 0, 0, now.Location()) + + //When + o := ObjectStorage{ + database: &db.Database{}, + } + err := o.getBillingDate(ctx) + + // Then + assert.NoError(t, err) + assert.Equal(t, o.database.BillingDate, expected) + }) +} + +func TestObjectStorage_GetAggregatedBuckets(t *testing.T) { + tests := map[string]struct { + givenSosBucketsUsage []oapi.SosBucketUsage + givenBucketDetails []BucketDetail + expectedAggregatedBuckets map[string]db.AggregatedBucket + }{ + "GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectAggregatedBucketObjects": { + givenSosBucketsUsage: []oapi.SosBucketUsage{ + createSosBucketUsage("bucket-test-1", 1), + createSosBucketUsage("bucket-test-2", 4), + createSosBucketUsage("bucket-test-3", 9), + createSosBucketUsage("bucket-test-4", 0), + createSosBucketUsage("bucket-test-5", 5), + }, + givenBucketDetails: []BucketDetail{ + createBucketDetail("bucket-test-1", "default", "orgA"), + createBucketDetail("bucket-test-2", "alpha", "orgB"), + createBucketDetail("bucket-test-3", "alpha", "orgB"), + createBucketDetail("bucket-test-4", "omega", "orgC"), + createBucketDetail("no-metrics-bucket", "beta", "orgD"), + }, + expectedAggregatedBuckets: map[string]db.AggregatedBucket{ + "default": createAggregatedBucket("orgA", 1), + "alpha": createAggregatedBucket("orgB", 13), + "omega": createAggregatedBucket("orgC", 0), + }, + }, + "GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectNoAggregatedBucketObjects": { + givenSosBucketsUsage: []oapi.SosBucketUsage{ + createSosBucketUsage("bucket-test-1", 1), + createSosBucketUsage("bucket-test-2", 4), + }, + givenBucketDetails: []BucketDetail{ + createBucketDetail("bucket-test-3", "default", "orgA"), + createBucketDetail("bucket-test-4", "alpha", "orgB"), + createBucketDetail("bucket-test-5", "alpha", "orgB"), + }, + expectedAggregatedBuckets: map[string]db.AggregatedBucket{}, + }, + "GivenSosBucketsUsageAndBuckets_WhenSosBucketsUsageEmpty_ThenExpectNoAggregatedBucketObjects": { + givenSosBucketsUsage: []oapi.SosBucketUsage{ + createSosBucketUsage("bucket-test-1", 1), + createSosBucketUsage("bucket-test-2", 4), + }, + givenBucketDetails: []BucketDetail{}, + expectedAggregatedBuckets: map[string]db.AggregatedBucket{}, + }, + "GivenSosBucketsUsageAndBuckets_WhenNoBuckets_ThenExpectNoAggregatedBucketObjects": { + givenSosBucketsUsage: []oapi.SosBucketUsage{}, + givenBucketDetails: []BucketDetail{ + createBucketDetail("bucket-test-3", "default", "orgA"), + createBucketDetail("bucket-test-4", "alpha", "orgB"), + }, + expectedAggregatedBuckets: map[string]db.AggregatedBucket{}, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + // Given + ctx := context.Background() + + // When + aggregatedBuckets := getAggregatedBuckets(ctx, tc.givenSosBucketsUsage, tc.givenBucketDetails) + + // Then + assert.Equal(t, aggregatedBuckets, tc.expectedAggregatedBuckets) + }) + } +} + +func TestObjectStorage_AadOrgAndNamespaceToBucket(t *testing.T) { + tests := map[string]struct { + givenBucketList exoscalev1.BucketList + expectedBucketDetails []BucketDetail + }{ + "GivenBucketListFromExoscale_WhenOrgAndNamespaces_ThenExpectBucketDetailObjects": { + givenBucketList: exoscalev1.BucketList{ + Items: []exoscalev1.Bucket{ + createBucket("bucket-1", "alpha", "orgA"), + createBucket("bucket-2", "beta", "orgB"), + createBucket("bucket-3", "alpha", "orgA"), + createBucket("bucket-4", "omega", "orgB"), + createBucket("bucket-5", "theta", "orgC"), + }, + }, + expectedBucketDetails: []BucketDetail{ + createBucketDetail("bucket-1", "alpha", "orgA"), + createBucketDetail("bucket-2", "beta", "orgB"), + createBucketDetail("bucket-3", "alpha", "orgA"), + createBucketDetail("bucket-4", "omega", "orgB"), + createBucketDetail("bucket-5", "theta", "orgC"), + }, + }, + "GivenBucketListFromExoscale_WhenNoOrgOrNamespaces_ThenExpectNoBucketDetailObjects": { + givenBucketList: exoscalev1.BucketList{ + Items: []exoscalev1.Bucket{ + createBucket("bucket-1", "", "orgA"), + createBucket("bucket-2", "beta", ""), + createBucket("bucket-3", "", ""), + }, + }, + expectedBucketDetails: []BucketDetail{}, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + // Given + ctx := context.Background() + + // When + bucketDetails := addOrgAndNamespaceToBucket(ctx, tc.givenBucketList) + + // Then + assert.Equal(t, tc.expectedBucketDetails, bucketDetails) + }) + } +} + +func createBucket(name, namespace, organization string) exoscalev1.Bucket { + labels := make(map[string]string) + if namespace != "" { + labels[namespaceLabel] = namespace + } + if organization != "" { + labels[organizationLabel] = organization + } + return exoscalev1.Bucket{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: exoscalev1.BucketSpec{ + ForProvider: exoscalev1.BucketParameters{ + BucketName: name, + }, + }, + } +} + +func createAggregatedBucket(organization string, size float64) db.AggregatedBucket { + return db.AggregatedBucket{ + Organization: organization, + StorageUsed: size, + } +} + +func createBucketDetail(bucketName, namespace, organization string) BucketDetail { + return BucketDetail{ + Organization: organization, + BucketName: bucketName, + Namespace: namespace, + } +} + +func createSosBucketUsage(bucketName string, size int) oapi.SosBucketUsage { + date := time.Now() + actualSize := int64(size) + zone := oapi.ZoneName("ch-gva-2") + return oapi.SosBucketUsage{ + CreatedAt: &date, + Name: &bucketName, + Size: &actualSize, + ZoneName: &zone, + } +} From 1fac4a144daf0f3903988cae8d895d9ea3b2c910 Mon Sep 17 00:00:00 2001 From: Gabriel Saratura Date: Thu, 6 Oct 2022 15:41:26 +0300 Subject: [PATCH 3/3] Add docs --- .../assets/images/application-logic.drawio.svg | 4 ++++ .../ROOT/pages/explanations/data-usage.adoc | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docs/modules/ROOT/assets/images/application-logic.drawio.svg create mode 100644 docs/modules/ROOT/pages/explanations/data-usage.adoc diff --git a/docs/modules/ROOT/assets/images/application-logic.drawio.svg b/docs/modules/ROOT/assets/images/application-logic.drawio.svg new file mode 100644 index 0000000..ea1633b --- /dev/null +++ b/docs/modules/ROOT/assets/images/application-logic.drawio.svg @@ -0,0 +1,4 @@ + + + +
Exoscale buckets with usage size
Exoscale buckets...
K8s buckets with namespace and organisation
K8s buckets with...
Match by name
Match by name
postgres billing database
postgres billin...
Aggregate usage storage by namespace
Aggregate usage stor...Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/modules/ROOT/pages/explanations/data-usage.adoc b/docs/modules/ROOT/pages/explanations/data-usage.adoc new file mode 100644 index 0000000..f4be2c4 --- /dev/null +++ b/docs/modules/ROOT/pages/explanations/data-usage.adoc @@ -0,0 +1,17 @@ += Data Usage + +This page gives a brief overview how buckets data usage is saved to the postgres billing database. + +== Data flow + +image::application-logic.drawio.svg[] + +== Data source +- Buckets are fetched from Exoscale Provider and K8s Cluster. +- The bucket names in Exoscale are unique across organisation which prevents clusters having same bucket names. + +== Data saving + +- The data is saved in a postgres database. +- A database must be provided in the postgres URL. +- The application ensures the initial configuration of specified database in the URL.