From f0bab7181599b0663fa5d17257c622e79604bba8 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 15:09:21 +0300 Subject: [PATCH 01/71] Correct regex to match current version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19c5ae85..34891de3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def load_long_description(): def changes(): changelog = path.join(PKG_DIR, 'CHANGELOG.rst') pattern = ( - r'(`(v\d+.\d+.\d+)`_( - \d+.-\w+-\d{4}\r?\n-+\r?\n.*?))' + r'(`(v\d+.\d+.\d+)`_( - \d{1,2}-\w+-\d{4}\r?\n-+\r?\n.*?))' r'\r?\n\r?\n\r?\n`v\d+.\d+.\d+`_' ) result = re.search(pattern, read_file(changelog), re.S) From 4e327ec3442bf8ded9b245461dda9f7fbfe0b0a7 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 15:14:10 +0300 Subject: [PATCH 02/71] Correct change log --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b6265ffc..0e55de02 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -183,7 +183,7 @@ Added - Initial release -.. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...develop +.. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 .. _v0.4.5: https://github.com/joke2k/django-environ/compare/v0.4.4...v0.4.5 .. _v0.4.4: https://github.com/joke2k/django-environ/compare/v0.4.3...v0.4.4 From d0d903821fad448705d6b239efe81349b8b58e5f Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 15:23:45 +0300 Subject: [PATCH 03/71] Bump version --- environ/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/__init__.py b/environ/__init__.py index e307a317..bf8689a0 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.6.0' +__version__ = '0.7.0' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' From 1d817ff5446c963a8bf21af4ec2676bd4e14e465 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 15:14:10 +0300 Subject: [PATCH 04/71] Update change log --- CHANGELOG.rst | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e55de02..3b251fae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is *inspired* by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.7.0`_ - 00-Unreleased-2021 +------------------------------ +Changed ++++++++ + - Added 'Funding' and 'Say Thanks!' project urls on pypi. + + `v0.6.0`_ - 4-September-2021 ---------------------------- Added @@ -160,14 +167,14 @@ Changed +++++++ - Rewriting README.rst -v0.2.1 19-April-2013 --------------------- +v0.2.1 - 19-April-2013 +---------------------- Changed +++++++ - ``Env.__call__`` now uses ``Env.get_value`` instance method -v0.2 16-April-2013 ------------------- +v0.2 - 16-April-2013 +-------------------- Added +++++ - Add advanced float parsing (comma and dot symbols to separate thousands and decimals) @@ -176,13 +183,14 @@ Fixed +++++ - Fixed typos in the documentation -v0.1 2-April-2013 ------------------ +v0.1 - 2-April-2013 +------------------- Added +++++ - Initial release +.. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...develop .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 .. _v0.4.5: https://github.com/joke2k/django-environ/compare/v0.4.4...v0.4.5 From 9caeaa5de8dde5710b321725a4b51e7b3eef571e Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 15:33:40 +0300 Subject: [PATCH 05/71] Add 'Funding' and 'Say Thanks!' project urls --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 34891de3..24e8a85f 100644 --- a/setup.py +++ b/setup.py @@ -187,7 +187,9 @@ def get_version_string(): # Project's URLs PROJECT_URLS = { - 'Documentation': 'https://django-environ.readthedocs.io', + 'Documentation': find_meta('url'), + 'Funding': 'https://opencollective.com/django-environ', + 'Say Thanks!': 'https://saythanks.io/to/joke2k', 'Changelog': '{}/en/latest/changelog.html'.format(find_meta('url')), 'Bug Tracker': 'https://github.com/joke2k/django-environ/issues', 'Source Code': 'https://github.com/joke2k/django-environ', From 4c49ce0fcf04a4d609ffccdebc4dc8ebf599cfbb Mon Sep 17 00:00:00 2001 From: Ben Wilber Date: Sat, 4 Sep 2021 18:44:46 +0300 Subject: [PATCH 06/71] Support negative float strings --- environ/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 9561f190..29b5f7f6 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -413,10 +413,10 @@ def parse_value(cls, value, cast): value = tuple([x for x in val if x]) elif cast is float: # clean string - float_str = re.sub(r'[^\d,\.]', '', value) + float_str = re.sub(r'[^\d,.-]', '', value) # split for avoid thousand separator and different # locale comma/dot symbol - parts = re.split(r'[,\.]', float_str) + parts = re.split(r'[,.]', float_str) if len(parts) == 1: float_str = parts[0] else: From 8391cbbf59ad4d579733e1fc54b68350729fac31 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 18:45:10 +0300 Subject: [PATCH 07/71] Amended tests to cover 4c49ce0 --- tests/fixtures.py | 1 + tests/test_env.py | 20 +++++++++++++------- tests/test_env.txt | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index a476e298..2de0c395 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -36,6 +36,7 @@ def generateData(cls): FLOAT_COMMA_VAR='33,3', FLOAT_STRANGE_VAR1='123,420,333.3', FLOAT_STRANGE_VAR2='123.420.333,3', + FLOAT_NEGATIVE_VAR='-1.0', BOOL_TRUE_VAR='1', BOOL_TRUE_VAR2='True', BOOL_FALSE_VAR='0', diff --git a/tests/test_env.py b/tests/test_env.py index c4807de0..5f76417f 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -75,13 +75,19 @@ def test_int(self): def test_int_with_none_default(self): assert self.env('NOT_PRESENT_VAR', cast=int, default=None) is None - def test_float(self): - assert_type_and_value(float, 33.3, self.env('FLOAT_VAR', cast=float)) - assert_type_and_value(float, 33.3, self.env.float('FLOAT_VAR')) - - assert_type_and_value(float, 33.3, self.env('FLOAT_COMMA_VAR', cast=float)) - assert_type_and_value(float, 123420333.3, self.env('FLOAT_STRANGE_VAR1', cast=float)) - assert_type_and_value(float, 123420333.3, self.env('FLOAT_STRANGE_VAR2', cast=float)) + @pytest.mark.parametrize( + 'value,variable,cast', + [ + (33.3, 'FLOAT_VAR'), + (33.3, 'FLOAT_COMMA_VAR'), + (123420333.3, 'FLOAT_STRANGE_VAR1'), + (123420333.3, 'FLOAT_STRANGE_VAR2'), + (-1.0, 'FLOAT_NEGATIVE_VAR'), + ] + ) + def test_float(self, value, variable, cast): + assert_type_and_value(float, value, self.env.float(variable)) + assert_type_and_value(float, value, self.env(variable, cast=float)) def test_bool_true(self): assert_type_and_value(bool, True, self.env('BOOL_TRUE_VAR', cast=bool)) diff --git a/tests/test_env.txt b/tests/test_env.txt index f00ce490..711639f1 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -17,6 +17,7 @@ FLOAT_VAR=33.3 FLOAT_COMMA_VAR=33,3 FLOAT_STRANGE_VAR1=123,420,333.3 FLOAT_STRANGE_VAR2=123.420.333,3 +FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR EMPTY_LIST= INT_VAR=42 From b21219063615b1c611d5bdcc16476d02a86682f6 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 18:48:01 +0300 Subject: [PATCH 08/71] Update change log --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b251fae..58886099 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ and this project adheres to `Semantic Versioning `_) + Changed +++++++ - Added 'Funding' and 'Say Thanks!' project urls on pypi. From 4cc2bbbc8af1c09fd37a972baa1df7e9b13150dc Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 18:51:31 +0300 Subject: [PATCH 09/71] Fix test for 4c49ce0 --- tests/test_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_env.py b/tests/test_env.py index 5f76417f..b85a539b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -76,7 +76,7 @@ def test_int_with_none_default(self): assert self.env('NOT_PRESENT_VAR', cast=int, default=None) is None @pytest.mark.parametrize( - 'value,variable,cast', + 'value,variable', [ (33.3, 'FLOAT_VAR'), (33.3, 'FLOAT_COMMA_VAR'), @@ -85,7 +85,7 @@ def test_int_with_none_default(self): (-1.0, 'FLOAT_NEGATIVE_VAR'), ] ) - def test_float(self, value, variable, cast): + def test_float(self, value, variable): assert_type_and_value(float, value, self.env.float(variable)) assert_type_and_value(float, value, self.env(variable, cast=float)) From 6a87fa716957320221eb1ade9f27c683491251a2 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 4 Sep 2021 23:49:23 +0300 Subject: [PATCH 10/71] Add tests for #118 --- tests/fixtures.py | 17 +++++++++++++---- tests/test_env.py | 36 +++++++++++++++++++++++++----------- tests/test_env.txt | 17 +++++++++++++---- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 2de0c395..db70501b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -37,10 +37,19 @@ def generateData(cls): FLOAT_STRANGE_VAR1='123,420,333.3', FLOAT_STRANGE_VAR2='123.420.333,3', FLOAT_NEGATIVE_VAR='-1.0', - BOOL_TRUE_VAR='1', - BOOL_TRUE_VAR2='True', - BOOL_FALSE_VAR='0', - BOOL_FALSE_VAR2='False', + BOOL_TRUE_STRING_LIKE_INT='1', + BOOL_TRUE_INT=1, + BOOL_TRUE_STRING_LIKE_BOOL='True', + BOOL_TRUE_STRING_1='on', + BOOL_TRUE_STRING_2='ok', + BOOL_TRUE_STRING_3='yes', + BOOL_TRUE_STRING_4='y', + BOOL_TRUE_STRING_5='true', + BOOL_TRUE_BOOL=True, + BOOL_FALSE_STRING_LIKE_INT='0', + BOOL_FALSE_INT=0, + BOOL_FALSE_STRING_LIKE_BOOL='False', + BOOL_FALSE_BOOL=False, PROXIED_VAR='$STR_VAR', INT_LIST='42,33', INT_TUPLE='(42,33)', diff --git a/tests/test_env.py b/tests/test_env.py index b85a539b..b3040954 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -89,15 +89,27 @@ def test_float(self, value, variable): assert_type_and_value(float, value, self.env.float(variable)) assert_type_and_value(float, value, self.env(variable, cast=float)) - def test_bool_true(self): - assert_type_and_value(bool, True, self.env('BOOL_TRUE_VAR', cast=bool)) - assert_type_and_value(bool, True, self.env('BOOL_TRUE_VAR2', cast=bool)) - assert_type_and_value(bool, True, self.env.bool('BOOL_TRUE_VAR')) - - def test_bool_false(self): - assert_type_and_value(bool, False, self.env('BOOL_FALSE_VAR', cast=bool)) - assert_type_and_value(bool, False, self.env('BOOL_FALSE_VAR2', cast=bool)) - assert_type_and_value(bool, False, self.env.bool('BOOL_FALSE_VAR')) + @pytest.mark.parametrize( + 'value,variable', + [ + (True, 'BOOL_TRUE_STRING_LIKE_INT'), + (True, 'BOOL_TRUE_STRING_LIKE_BOOL'), + (True, 'BOOL_TRUE_INT'), + (True, 'BOOL_TRUE_BOOL'), + (True, 'BOOL_TRUE_STRING_1'), + (True, 'BOOL_TRUE_STRING_2'), + (True, 'BOOL_TRUE_STRING_3'), + (True, 'BOOL_TRUE_STRING_4'), + (True, 'BOOL_TRUE_STRING_5'), + (False, 'BOOL_FALSE_STRING_LIKE_INT'), + (False, 'BOOL_FALSE_INT'), + (False, 'BOOL_FALSE_STRING_LIKE_BOOL'), + (False, 'BOOL_FALSE_BOOL'), + ] + ) + def test_bool_true(self, value, variable): + assert_type_and_value(bool, value, self.env.bool(variable)) + assert_type_and_value(bool, value, self.env(variable, cast=bool)) def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' @@ -256,8 +268,10 @@ def test_path(self): def test_smart_cast(self): assert self.env.get_value('STR_VAR', default='string') == 'bar' - assert self.env.get_value('BOOL_TRUE_VAR', default=True) - assert self.env.get_value('BOOL_FALSE_VAR', default=True) is False + assert self.env.get_value('BOOL_TRUE_STRING_LIKE_INT', default=True) + assert not self.env.get_value( + 'BOOL_FALSE_STRING_LIKE_INT', + default=True) assert self.env.get_value('INT_VAR', default=1) == 42 assert self.env.get_value('FLOAT_VAR', default=1.2) == 33.3 diff --git a/tests/test_env.txt b/tests/test_env.txt index 711639f1..dfbd61ef 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -1,5 +1,4 @@ DICT_VAR=foo=bar,test=on -BOOL_FALSE_VAR2=False DATABASE_MYSQL_URL=mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true DATABASE_MYSQL_GIS_URL=mysqlgis://user:password@127.0.0.1/some_database CACHE_URL=memcache://127.0.0.1:11211 @@ -7,11 +6,21 @@ CACHE_REDIS=rediscache://127.0.0.1:6379/1?client_class=django_redis.client.Defau EMAIL_URL=smtps://user@domain.com:password@smtp.example.com:587 URL_VAR=http://www.google.com/ PATH_VAR=/home/dev -BOOL_FALSE_VAR=0 -BOOL_TRUE_VAR2=True +BOOL_TRUE_STRING_LIKE_INT='1' +BOOL_TRUE_INT=1 +BOOL_TRUE_STRING_LIKE_BOOL='True' +BOOL_TRUE_STRING_1='on' +BOOL_TRUE_STRING_2='ok' +BOOL_TRUE_STRING_3='yes' +BOOL_TRUE_STRING_4='y' +BOOL_TRUE_STRING_5='true' +BOOL_TRUE_BOOL=True +BOOL_FALSE_STRING_LIKE_INT='0' +BOOL_FALSE_INT=0 +BOOL_FALSE_STRING_LIKE_BOOL='False' +BOOL_FALSE_BOOL=False DATABASE_SQLITE_URL=sqlite:////full/path/to/your/database/file.sqlite JSON_VAR={"three": 33.44, "two": 2, "one": "bar"} -BOOL_TRUE_VAR=1 DATABASE_URL=postgres://uf07k1:wegauwhg@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722 FLOAT_VAR=33.3 FLOAT_COMMA_VAR=33,3 From 3d2a99041e425524ca3e514e8ba47c4bc49bb792 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 02:32:41 +0300 Subject: [PATCH 11/71] Fix links in the documentation --- .github/workflows/docs.yml | 3 +++ BACKERS.rst | 15 ++++++--------- CHANGELOG.rst | 12 +++++++----- README.rst | 10 +++++----- docs/tips.rst | 2 +- environ/environ.py | 2 +- tox.ini | 21 ++++++++++++++++++++- 7 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f2bda872..7dba5edb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,5 +33,8 @@ jobs: python -m pip install --upgrade pip pip install tox tox-gh-actions + - name: Check external links in the package documentation + run: tox -e linkcheck + - name: Build and test package documentation run: tox -e docs diff --git a/BACKERS.rst b/BACKERS.rst index aa2a8148..4ac3b73c 100644 --- a/BACKERS.rst +++ b/BACKERS.rst @@ -9,9 +9,9 @@ Sponsors -------- Support this project by becoming a sponsor. Your logo will show up here with a -link to your website. `Became sponsor `_. +link to your website. `Became sponsor `_. -|ocsponsor0| |ocsponsor1| |ocsponsor2| +|ocsponsor0| |ocsponsor1| Backers ------- @@ -21,14 +21,11 @@ Thank you to all our backers! |ocbackerimage| .. |ocsponsor0| image:: https://opencollective.com/django-environ/sponsor/0/avatar.svg - :target: https://opencollective.com/django-environ/sponsor/0/website - :alt: Sponsor -.. |ocsponsor1| image:: https://opencollective.com/django-environ/sponsor/1/avatar.svg - :target: https://opencollective.com/django-environ/sponsor/1/website - :alt: Sponsor -.. |ocsponsor2| image:: https://opencollective.com/django-environ/sponsor/2/avatar.svg - :target: https://opencollective.com/django-environ/sponsor/2/website + :target: https://triplebyte.com/ :alt: Sponsor +.. |ocsponsor1| image:: https://images.opencollective.com/static/images/become_sponsor.svg + :target: https://opencollective.com/django-environ/contribute/sponsors-3474/checkout + :alt: Become a Sponsor .. |ocbackerimage| image:: https://opencollective.com/django-environ/backers.svg?width=890 :target: https://opencollective.com/django-environ :alt: Backers on Open Collective diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 58886099..ca1ce302 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ Changelog ========= All notable changes to this project will be documented in this file. -The format is *inspired* by `Keep a Changelog `_ -and this project adheres to `Semantic Versioning `_. +The format is *inspired* by `Keep a Changelog `_ +and this project adheres to `Semantic Versioning `_. `v0.7.0`_ - 00-Unreleased-2021 ------------------------------ @@ -12,6 +12,10 @@ Added - Added support for negative float strings (`#160 `_) +Fixed ++++++ + - Fixed links in the documentation + Changed +++++++ - Added 'Funding' and 'Say Thanks!' project urls on pypi. @@ -202,9 +206,7 @@ Added .. _v0.4.4: https://github.com/joke2k/django-environ/compare/v0.4.3...v0.4.4 .. _v0.4.3: https://github.com/joke2k/django-environ/compare/v0.4.2...v0.4.3 .. _v0.4.2: https://github.com/joke2k/django-environ/compare/v0.4.1...v0.4.2 -.. _v0.4.1: https://github.com/joke2k/django-environ/compare/v0.4.0...v0.4.1 +.. _v0.4.1: https://github.com/joke2k/django-environ/compare/v0.4...v0.4.1 .. _v0.4: https://github.com/joke2k/django-environ/compare/v0.3.1...v0.4 .. _v0.3.1: https://github.com/joke2k/django-environ/compare/v0.3...v0.3.1 .. _v0.3: https://github.com/joke2k/django-environ/compare/v0.2.1...v0.3 -.. _`Keep a Changelog`: http://keepachangelog.com/en/1.0.0/ -.. _`Semantic Versioning`: http://semver.org/spec/v2.0.0.html diff --git a/README.rst b/README.rst index 4a9b8e98..df5a31ca 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ .. -teaser-begin- ``django-environ`` is the Python package that allows you to use -`Twelve-factor methodology `_ to configure your +`Twelve-factor methodology `_ to configure your Django application with environment variables. .. -teaser-end- @@ -92,13 +92,13 @@ environment variables obtained from an environment file and provided by the OS: The idea of this package is to unify a lot of packages that make the same stuff: Take a string from ``os.environ``, parse and cast it to some of useful python -typed variables. To do that and to use the `12factor `_ +typed variables. To do that and to use the `12factor `_ approach, some connection strings are expressed as url, so this package can parse it and return a ``urllib.parse.ParseResult``. These strings from ``os.environ`` are loaded from a ``.env`` file and filled in ``os.environ`` with ``setdefault`` method, to avoid to overwrite the real environ. -A similar approach is used in `Two Scoops of Django `_ -book and explained in `12factor-django `_ +A similar approach is used in `Two Scoops of Django `_ +book and explained in `12factor-django `_ article. @@ -120,7 +120,7 @@ Project Information =================== ``django-environ`` is released under the `MIT / X11 License `__, -its documentation lives at `Read the Docs `_, +its documentation lives at `Read the Docs `_, the code on `GitHub `_, and the latest release on `PyPI `_. diff --git a/docs/tips.rst b/docs/tips.rst index f82b0c4c..20a8b360 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -29,7 +29,7 @@ To disable it use ``env.smart_cast = False``. Multiple redis cache locations ============================== -For redis cache, `multiple master/slave or shard locations `_ can be configured as follows: +For redis cache, multiple master/slave or shard locations can be configured as follows: .. code-block:: shell diff --git a/environ/environ.py b/environ/environ.py index 29b5f7f6..310b7481 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -728,7 +728,7 @@ def read_env(cls, env_file=None, **overrides): called read_env. Refs: - - http://www.wellfireinteractive.com/blog/easier-12-factor-django/ + - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 """ if env_file is None: diff --git a/tox.ini b/tox.ini index a32a27ea..b9c16244 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ minversion = 3.22 envlist = build coverage-report + linkcheck docs lint manifest @@ -69,6 +70,24 @@ commands_pre = python -m pip install . commands = flake8 environ setup.py +[testenv:linkcheck] +description = Check external links in the package documentation +# Keep basepython in sync with .readthedocs.yml and docs.yml +# (GitHub Action Workflow). +basepython = python3.8 +extras = docs +commands = + {envpython} -m sphinx \ + -j auto \ + -b linkcheck \ + {tty:--color} \ + -n -W -a \ + --keep-going \ + -d {envtmpdir}/doctrees \ + docs \ + docs/_build/linkcheck +isolated_build = true + [testenv:docs] description = Build package documentation (HTML) # Keep basepython in sync with .readthedocs.yml and docs.yml @@ -78,7 +97,7 @@ extras = docs commands = sphinx-build -n -T -W -b html -d {envtmpdir}/doctrees docs docs/_build/html sphinx-build -n -T -W -b doctest -d {envtmpdir}/doctrees docs docs/_build/html - python -m doctest AUTHORS.rst CHANGELOG.rst CONTRIBUTING.rst README.rst SECURITY.rst + {envpython} -m doctest AUTHORS.rst CHANGELOG.rst CONTRIBUTING.rst README.rst SECURITY.rst [testenv:manifest] description = Check MANIFEST.in in a source package for completeness From 5a952936c79a925902067d1a433185dd0ad956c6 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 02:39:38 +0300 Subject: [PATCH 12/71] Reformat docs tox target --- tox.ini | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index b9c16244..4877c140 100644 --- a/tox.ini +++ b/tox.ini @@ -95,9 +95,30 @@ description = Build package documentation (HTML) basepython = python3.8 extras = docs commands = - sphinx-build -n -T -W -b html -d {envtmpdir}/doctrees docs docs/_build/html - sphinx-build -n -T -W -b doctest -d {envtmpdir}/doctrees docs docs/_build/html - {envpython} -m doctest AUTHORS.rst CHANGELOG.rst CONTRIBUTING.rst README.rst SECURITY.rst + {envpython} -m sphinx \ + -j auto \ + -b html \ + {tty:--color} \ + -n -T -W \ + -d {envtmpdir}/doctrees \ + docs \ + docs/_build/html + + {envpython} -m sphinx \ + -j auto \ + -b doctest \ + {tty:--color} \ + -n -T -W \ + -d {envtmpdir}/doctrees \ + docs \ + docs/_build/doctest + + {envpython} -m doctest \ + AUTHORS.rst \ + CHANGELOG.rst \ + CONTRIBUTING.rst \ + README.rst \ + SECURITY.rst [testenv:manifest] description = Check MANIFEST.in in a source package for completeness From 05b7ae64ffd231eb74fe49fdddc2b9ae177a2e9e Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 02:39:56 +0300 Subject: [PATCH 13/71] Archive docs artifacts --- .github/workflows/docs.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7dba5edb..2c31930a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,3 +38,13 @@ jobs: - name: Build and test package documentation run: tox -e docs + + - name: Archive docs artifacts + if: always() + uses: actions/upload-artifact@v2 + with: + name: docs + path: docs + # Artifacts are retained for 90 days by default. + # In fact, we don't need such long period. + retention-days: 60 From 0c693fe93c19209e396acdcc52806dd9e206c2b2 Mon Sep 17 00:00:00 2001 From: Yuki Takino Date: Sun, 5 Sep 2021 11:23:58 +0300 Subject: [PATCH 14/71] Add elasticsearch5 to search scheme --- docs/types.rst | 2 ++ environ/environ.py | 22 +++++++++++++--------- tests/conftest.py | 6 ------ tests/test_search.py | 20 +++++++++++++++----- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/types.rst b/docs/types.rst index b2144d06..6bf12414 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -41,6 +41,8 @@ Supported types * ``search_url`` * Elasticsearch: ``elasticsearch://`` + * Elasticsearch2: ``elasticsearch2://`` + * Elasticsearch5: ``elasticsearch5://`` * Solr: ``solr://`` * Whoosh: ``whoosh://`` * Xapian: ``xapian://`` diff --git a/environ/environ.py b/environ/environ.py index 310b7481..5d897d52 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -30,11 +30,11 @@ try: from os import PathLike + Openable = (str, PathLike) except ImportError: Openable = (str,) - logger = logging.getLogger(__name__) @@ -64,7 +64,6 @@ def __repr__(self): class Env: - """Provide scheme-based lookups of environment variables so that each caller doesn't have to pass in `cast` and `default` parameters. @@ -79,9 +78,11 @@ class Env: NOTSET = NoValue() BOOLEAN_TRUE_STRINGS = ('true', 'on', 'ok', 'y', 'yes', '1') URL_CLASS = ParseResult - DEFAULT_DATABASE_ENV = 'DATABASE_URL' POSTGRES_FAMILY = ['postgres', 'postgresql', 'psql', 'pgsql', 'postgis'] + ELASTICSEARCH_FAMILY = ['elasticsearch' + x for x in ['', '2', '5']] + + DEFAULT_DATABASE_ENV = 'DATABASE_URL' DB_SCHEMES = { 'postgres': DJANGO_POSTGRES, 'postgresql': DJANGO_POSTGRES, @@ -146,6 +147,8 @@ class Env: "ElasticsearchSearchEngine", "elasticsearch2": "haystack.backends.elasticsearch2_backend." "Elasticsearch2SearchEngine", + "elasticsearch5": "haystack.backends.elasticsearch5_backend." + "Elasticsearch5SearchEngine", "solr": "haystack.backends.solr_backend.SolrEngine", "whoosh": "haystack.backends.whoosh_backend.WhooshEngine", "xapian": "haystack.backends.xapian_backend.XapianEngine", @@ -262,6 +265,7 @@ def db_url(self, var=DEFAULT_DATABASE_ENV, default=NOTSET, engine=None): self.get_value(var, default=default), engine=engine ) + db = db_url def cache_url(self, var=DEFAULT_CACHE_ENV, default=NOTSET, backend=None): @@ -275,6 +279,7 @@ def cache_url(self, var=DEFAULT_CACHE_ENV, default=NOTSET, backend=None): self.url(var, default=default), backend=backend ) + cache = cache_url def email_url(self, var=DEFAULT_EMAIL_ENV, default=NOTSET, backend=None): @@ -288,6 +293,7 @@ def email_url(self, var=DEFAULT_EMAIL_ENV, default=NOTSET, backend=None): self.url(var, default=default), backend=backend ) + email = email_url def search_url(self, var=DEFAULT_SEARCH_ENV, default=NOTSET, engine=None): @@ -492,7 +498,7 @@ def db_url_config(cls, url, engine=None): if url.scheme == 'oracle': # Django oracle/base.py strips port and fails on non-string value if not config['PORT']: - del(config['PORT']) + del (config['PORT']) else: config['PORT'] = str(config['PORT']) @@ -646,7 +652,7 @@ def search_url_config(cls, url, engine=None): config["ENGINE"] = cls.SEARCH_SCHEMES[url.scheme] # check commons params - params = {} + params = {} # type: dict if url.query: params = parse_qs(url.query) if 'EXCLUDED_INDEXES' in params.keys(): @@ -665,7 +671,7 @@ def search_url_config(cls, url, engine=None): if url.scheme == 'simple': return config - elif url.scheme in ['solr', 'elasticsearch', 'elasticsearch2']: + elif url.scheme in ['solr'] + cls.ELASTICSEARCH_FAMILY: if 'KWARGS' in params.keys(): config['KWARGS'] = params['KWARGS'][0] @@ -681,8 +687,7 @@ def search_url_config(cls, url, engine=None): config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) return config - if url.scheme in ['elasticsearch', 'elasticsearch2']: - + if url.scheme in cls.ELASTICSEARCH_FAMILY: split = path.rsplit("/", 1) if len(split) > 1: @@ -776,7 +781,6 @@ def read_env(cls, env_file=None, **overrides): class Path: - """Inspired to Django Two-scoops, handling File Paths in Settings.""" def path(self, *paths, **kwargs): diff --git a/tests/conftest.py b/tests/conftest.py index 32a19cdf..38550f73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,12 +19,6 @@ def solr_url(): return 'solr://127.0.0.1:8983/solr' -@pytest.fixture -def elasticsearch_url(): - """Return Elasticsearch URL.""" - return 'elasticsearch://127.0.0.1:9200/index' - - @pytest.fixture def whoosh_url(): """Return Whoosh URL.""" diff --git a/tests/test_search.py b/tests/test_search.py index 99666d59..6bb11175 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -32,14 +32,24 @@ def test_solr_multicore_parsing(solr_url): assert 'PATH' not in url -def test_elasticsearch_parsing(elasticsearch_url): +@pytest.mark.parametrize( + 'url,engine', + [ + ('elasticsearch://127.0.0.1:9200/index', + 'elasticsearch_backend.ElasticsearchSearchEngine'), + ('elasticsearch2://127.0.0.1:9200/index', + 'elasticsearch2_backend.Elasticsearch2SearchEngine'), + ('elasticsearch5://127.0.0.1:9200/index', + 'elasticsearch5_backend.Elasticsearch5SearchEngine'), + ] +) +def test_elasticsearch_parsing(url, engine): + """Ensure all supported Elasticsearch engines are recognized.""" timeout = 360 - url = '{}?TIMEOUT={}'.format(elasticsearch_url, timeout) + url = '{}?TIMEOUT={}'.format(url, timeout) url = Env.search_url_config(url) - assert url['ENGINE'] == ( - 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine' - ) + assert url['ENGINE'] == 'haystack.backends.{}'.format(engine) assert 'INDEX_NAME' in url.keys() assert url['INDEX_NAME'] == 'index' assert 'TIMEOUT' in url.keys() From 5e0b2b01089568f01df684c527c45d5fdbe783b2 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 11:25:49 +0300 Subject: [PATCH 15/71] Update change log --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca1ce302..11ecc370 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Added +++++ - Added support for negative float strings (`#160 `_) + - Added Elasticsearch5 to search scheme + (`#297 `_) Fixed +++++ From 3f21d84487e774205af301dcf0afcaf967820c88 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 11:45:35 +0300 Subject: [PATCH 16/71] Added Elasticsearch7 to search scheme --- CHANGELOG.rst | 3 +++ docs/types.rst | 1 + environ/environ.py | 4 +++- tests/fixtures.py | 2 +- tests/test_env.py | 4 ++-- tests/test_schema.py | 2 +- tests/test_search.py | 2 ++ 7 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 11ecc370..b2219412 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,9 @@ Added (`#160 `_) - Added Elasticsearch5 to search scheme (`#297 `_) + - Added Elasticsearch7 to search scheme + (`#314 `_) + Fixed +++++ diff --git a/docs/types.rst b/docs/types.rst index 6bf12414..1e80e582 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -43,6 +43,7 @@ Supported types * Elasticsearch: ``elasticsearch://`` * Elasticsearch2: ``elasticsearch2://`` * Elasticsearch5: ``elasticsearch5://`` + * Elasticsearch7: ``elasticsearch7://`` * Solr: ``solr://`` * Whoosh: ``whoosh://`` * Xapian: ``xapian://`` diff --git a/environ/environ.py b/environ/environ.py index 5d897d52..2dad620a 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -80,7 +80,7 @@ class Env: URL_CLASS = ParseResult POSTGRES_FAMILY = ['postgres', 'postgresql', 'psql', 'pgsql', 'postgis'] - ELASTICSEARCH_FAMILY = ['elasticsearch' + x for x in ['', '2', '5']] + ELASTICSEARCH_FAMILY = ['elasticsearch' + x for x in ['', '2', '5', '7']] DEFAULT_DATABASE_ENV = 'DATABASE_URL' DB_SCHEMES = { @@ -149,6 +149,8 @@ class Env: "Elasticsearch2SearchEngine", "elasticsearch5": "haystack.backends.elasticsearch5_backend." "Elasticsearch5SearchEngine", + "elasticsearch7": "haystack.backends.elasticsearch7_backend." + "Elasticsearch7SearchEngine", "solr": "haystack.backends.solr_backend.SolrEngine", "whoosh": "haystack.backends.whoosh_backend.WhooshEngine", "xapian": "haystack.backends.xapian_backend.XapianEngine", diff --git a/tests/fixtures.py b/tests/fixtures.py index db70501b..f685f5cb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -28,7 +28,7 @@ class FakeEnv: EXPORTED = 'exported var' @classmethod - def generateData(cls): + def generate_data(cls): return dict(STR_VAR='bar', MULTILINE_STR_VAR='foo\\nbar', INT_VAR='42', diff --git a/tests/test_env.py b/tests/test_env.py index b3040954..12fe9b48 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -26,7 +26,7 @@ def setup_method(self, method): class. setup_method is invoked for every test method of a class. """ self.old_environ = os.environ - os.environ = Env.ENVIRON = FakeEnv.generateData() + os.environ = Env.ENVIRON = FakeEnv.generate_data() self.env = Env() def teardown_method(self, method): @@ -334,7 +334,7 @@ def setup_method(self, method): """ super().setup_method(method) - self.CONFIG = FakeEnv.generateData() + self.CONFIG = FakeEnv.generate_data() class MyEnv(Env): ENVIRON = self.CONFIG diff --git a/tests/test_schema.py b/tests/test_schema.py index ba296823..7a7f62ec 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -19,7 +19,7 @@ def setup_module(): global _old_environ _old_environ = os.environ - os.environ = Env.ENVIRON = FakeEnv.generateData() + os.environ = Env.ENVIRON = FakeEnv.generate_data() def teardown_module(): diff --git a/tests/test_search.py b/tests/test_search.py index 6bb11175..a81471e5 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -41,6 +41,8 @@ def test_solr_multicore_parsing(solr_url): 'elasticsearch2_backend.Elasticsearch2SearchEngine'), ('elasticsearch5://127.0.0.1:9200/index', 'elasticsearch5_backend.Elasticsearch5SearchEngine'), + ('elasticsearch7://127.0.0.1:9200/index', + 'elasticsearch7_backend.Elasticsearch7SearchEngine'), ] ) def test_elasticsearch_parsing(url, engine): From 5602550c8e1903c31aa2c9862966260dc7082dfd Mon Sep 17 00:00:00 2001 From: Duarte Duarte Date: Sun, 5 Sep 2021 14:51:23 +0300 Subject: [PATCH 17/71] Use default option in Env.bytes() --- environ/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 2dad620a..70837a4b 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -193,7 +193,7 @@ def bytes(self, var, default=NOTSET, encoding='utf8'): """ :rtype: bytes """ - return self.get_value(var, cast=str).encode(encoding) + return self.get_value(var, cast=str, default=default).encode(encoding) def bool(self, var, default=NOTSET): """ From 7292c9c2e51bc30af13bd7be4594387e6379d8ab Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 14:53:13 +0300 Subject: [PATCH 18/71] Use encode in Env.bytes() only if possible --- environ/environ.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 70837a4b..17a6427c 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -193,7 +193,10 @@ def bytes(self, var, default=NOTSET, encoding='utf8'): """ :rtype: bytes """ - return self.get_value(var, cast=str, default=default).encode(encoding) + value = self.get_value(var, cast=str, default=default) + if hasattr(value, 'encode'): + return value.encode(encoding) + return value def bool(self, var, default=NOTSET): """ From d307b6867bdb814dc73a4f7251e4c1473b3a7828 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 14:53:38 +0300 Subject: [PATCH 19/71] Resolve proxied values for bytes --- environ/environ.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 17a6427c..b91d3ff8 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -363,8 +363,9 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): value = default # Resolve any proxied values - if hasattr(value, 'startswith') and value.startswith('$'): - value = value.lstrip('$') + prefix = b'$' if isinstance(value, bytes) else '$' + if hasattr(value, 'startswith') and value.startswith(prefix): + value = value.lstrip(prefix) value = self.get_value(value, cast=cast, default=default) # Smart casting From 15bc27a4556a55be31dea32fd937a9923eaf4890 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 14:54:10 +0300 Subject: [PATCH 20/71] Add test for 5602550 --- tests/test_env.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_env.py b/tests/test_env.py index 12fe9b48..efb09fbc 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -65,8 +65,16 @@ def test_str(self, var, val, multiline): assert self.env(var) == val assert self.env.str(var, multiline=multiline) == val - def test_bytes(self): - assert_type_and_value(bytes, b'bar', self.env.bytes('STR_VAR')) + @pytest.mark.parametrize( + 'var,val,default', + [ + ('STR_VAR', b'bar', Env.NOTSET), + ('NON_EXISTENT_BYTES_VAR', b'some-default', b'some-default'), + ('NON_EXISTENT_STR_VAR', b'some-default', 'some-default'), + ] + ) + def test_bytes(self, var, val, default): + assert_type_and_value(bytes, val, self.env.bytes(var, default=default)) def test_int(self): assert_type_and_value(int, 42, self.env('INT_VAR', cast=int)) From 09dbc01f5c8e577e9af8c7fba86e828b89665a63 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Sep 2021 14:56:22 +0300 Subject: [PATCH 21/71] Update change log --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2219412..3aafe600 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,11 +15,14 @@ Added (`#297 `_) - Added Elasticsearch7 to search scheme (`#314 `_) + - Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()`` Fixed +++++ - Fixed links in the documentation + - Use default option in ``Env.bytes()`` + (`#206 `_) Changed +++++++ @@ -34,6 +37,7 @@ Added - Django 3.1 and 3.2 are now supported - Added missed classifiers to ``setup.py`` - Accept Python 3.6 path-like objects for ``read_env`` + (`#106 `_, `#286 `_) Fixed +++++ @@ -41,6 +45,7 @@ Fixed - Fixed typos in the documentation - Added missed files to the package contents - Fixed ``db_url_config`` to work the same for all postgres-like schemes + (`#264 `_, `#268 `_) Changed +++++++ From ab217163c7d6378e20323306082cccaa60dd2744 Mon Sep 17 00:00:00 2001 From: Nick Cannariato <434063+nickcannariato@users.noreply.github.com> Date: Thu, 19 Mar 2020 02:03:25 -0500 Subject: [PATCH 22/71] Stop raising UserWarning if .env file isn't found Currently, if a `.env` file isn't found by the read_env() classmethod, it raises a UserWarning. Based on Python's logging guide, warnings.warn() is only appropriate to be raised in library code if the issue is avoidable and the client application should be modified to eliminate the warning. Conversely, it recommends logging.warning() if there is nothing the client application can do about the situation, but the event should still be noted (see ref1). In this particular case, the function is working with `.env` files that should only ever exist in development environments, which means that the function will erroneously warn the consumer in production and staging environments. With this in mind, I believe the most sensible thing to do would be to convert read_env from using the warnings module to using the logging module. Additionally, since the lack of a `.env` file is going to occur regularly during normal operation of the program, I also believe that this should be downgraded to an INFO level message from a WARNING level message. joke2k/django-environ#243 requests that the .env file be made optional, and as far as I can the .env file is indeed optional, it's just that a UserWarning is being raised instead of a logged message. With that in mind, I'm referencing it as being closed by this change. ref1: https://docs.python.org/3/howto/logging.html#when-to-use-logging fixes: joke2k/django-environ#243 --- environ/environ.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index b91d3ff8..0b97c5e9 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -749,7 +749,7 @@ def read_env(cls, env_file=None, **overrides): '.env' ) if not os.path.exists(env_file): - warnings.warn( + logger.info( "%s doesn't exist - if you're not configuring your " "environment separately, create one." % env_file) return @@ -762,8 +762,8 @@ def read_env(cls, env_file=None, **overrides): with env_file as f: content = f.read() except OSError: - warnings.warn( - "Error reading %s - if you're not configuring your " + logger.info( + "%s not found - if you're not configuring your " "environment separately, check this." % env_file) return From a0332b0214fb40ad04d7a369799a1f75f545cda3 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 6 Sep 2021 11:33:29 +0300 Subject: [PATCH 23/71] Update change log --- CHANGELOG.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3aafe600..6b980e3d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,7 +26,10 @@ Fixed Changed +++++++ - - Added 'Funding' and 'Say Thanks!' project urls on pypi. + - Added 'Funding' and 'Say Thanks!' project urls on pypi. + - Stop raising ``UserWarning`` if ``.env`` file isn't found. Log a message + with ``INFO`` log level instead. + (`#243 `_) `v0.6.0`_ - 4-September-2021 From 192b813fc97fd395cb432eba5d0064a73d4ab9f0 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 6 Sep 2021 11:54:12 +0300 Subject: [PATCH 24/71] Reformat change log --- CHANGELOG.rst | 174 ++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b980e3d..0e4bd831 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,103 +1,105 @@ Changelog ========= -All notable changes to this project will be documented in this file. -The format is *inspired* by `Keep a Changelog `_ +All notable changes to this project will be documented in this file. +The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. `v0.7.0`_ - 00-Unreleased-2021 ------------------------------ Added +++++ - - Added support for negative float strings - (`#160 `_) - - Added Elasticsearch5 to search scheme - (`#297 `_) - - Added Elasticsearch7 to search scheme - (`#314 `_) - - Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()`` +- Added support for negative float strings + `#160 `_. +- Added Elasticsearch5 to search scheme + `#297 `_. +- Added Elasticsearch7 to search scheme + `#314 `_. +- Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. Fixed +++++ - - Fixed links in the documentation - - Use default option in ``Env.bytes()`` - (`#206 `_) +- Fixed links in the documentation +- Use default option in ``Env.bytes()`` + `#206 `_. Changed +++++++ - - Added 'Funding' and 'Say Thanks!' project urls on pypi. - - Stop raising ``UserWarning`` if ``.env`` file isn't found. Log a message - with ``INFO`` log level instead. - (`#243 `_) +- Added 'Funding' and 'Say Thanks!' project urls on pypi. +- Stop raising ``UserWarning`` if ``.env`` file isn't found. Log a message with + ``INFO`` log level instead `#243 `_. `v0.6.0`_ - 4-September-2021 ---------------------------- Added +++++ - - Python 3.9, 3.10 and pypy 3.7 are now supported - - Django 3.1 and 3.2 are now supported - - Added missed classifiers to ``setup.py`` - - Accept Python 3.6 path-like objects for ``read_env`` - (`#106 `_, `#286 `_) +- Python 3.9, 3.10 and pypy 3.7 are now supported. +- Django 3.1 and 3.2 are now supported. +- Added missed classifiers to ``setup.py``. +- Accept Python 3.6 path-like objects for ``read_env`` + `#106 `_, + `#286 `_. Fixed +++++ - - Fixed various code linting errors - - Fixed typos in the documentation - - Added missed files to the package contents - - Fixed ``db_url_config`` to work the same for all postgres-like schemes - (`#264 `_, `#268 `_) +- Fixed various code linting errors. +- Fixed typos in the documentation. +- Added missed files to the package contents. +- Fixed ``db_url_config`` to work the same for all postgres-like schemes + `#264 `_, + `#268 `_. Changed +++++++ - - Refactor tests to use pytest and follow DRY - - Moved CI to GitHub Actions - - Restructuring of project documentation - - Build and test package documentation as a part of CI pipeline - - Build and test package distribution as a part of CI pipeline - - Check ``MANIFEST.in`` in a source package for completeness as a part of CI pipeline - - Added ``pytest`` and ``coverage[toml]`` to setuptools' ``extras_require`` +- Refactor tests to use pytest and follow DRY. +- Moved CI to GitHub Actions. +- Restructuring of project documentation. +- Build and test package documentation as a part of CI pipeline. +- Build and test package distribution as a part of CI pipeline. +- Check ``MANIFEST.in`` in a source package for completeness as a part of CI + pipeline. +- Added ``pytest`` and ``coverage[toml]`` to setuptools' ``extras_require``. `v0.5.0`_ - 30-August-2021 -------------------------- Added +++++ - - Support for Django 2.1 & 2.2 - - Added tox.ini targets - - Added secure redis backend URLs via ``rediss://`` - - Add ``cast=str`` to ``str()`` method +- Support for Django 2.1 & 2.2. +- Added tox.ini targets. +- Added secure redis backend URLs via ``rediss://``. +- Add ``cast=str`` to ``str()`` method. Fixed +++++ - - Fixed misspelling in the documentation +- Fixed misspelling in the documentation. Changed +++++++ - - Validate empty cache url and invalid cache schema - - Set ``long_description_content_type`` in setup - - Improved Django 1.11 database configuration support +- Validate empty cache url and invalid cache schema. +- Set ``long_description_content_type`` in setup. +- Improved Django 1.11 database configuration support. `v0.4.5`_ - 25-June-2018 ------------------------ Added +++++ - - Support for Django 2.0 - - Support for smart casting - - Support PostgreSQL unix domain socket paths - - Tip: Multiple env files +- Support for Django 2.0. +- Support for smart casting. +- Support PostgreSQL unix domain socket paths. +- Tip: Multiple env files. Changed +++++++ - - Fix parsing option values ``None``, ``True`` and ``False`` - - Order of importance of engine configuration in ``db_url_config`` +- Fix parsing option values ``None``, ``True`` and ``False``. +- Order of importance of engine configuration in ``db_url_config``. Removed +++++++ - - Remove ``django`` and ``six`` dependencies +- Remove ``django`` and ``six`` dependencies. `v0.4.4`_ - 21-August-2017 @@ -105,111 +107,115 @@ Removed Added +++++ - - Support for ``django-redis`` multiple locations (master/slave, shards) - - Support for Elasticsearch2 - - Support for Mysql-connector - - Support for ``pyodbc`` - - Add ``__contains__`` feature to Environ class +- Support for ``django-redis`` multiple locations (master/slave, shards). +- Support for Elasticsearch2. +- Support for Mysql-connector. +- Support for ``pyodbc``. +- Add ``__contains__`` feature to Environ class. Fixed +++++ - - Fix Path subtracting +- Fix Path subtracting. `v0.4.3`_ - 21-August-2017 -------------------------- Changed +++++++ - - Rollback the default Environ to ``os.environ`` +- Rollback the default Environ to ``os.environ``. `v0.4.2`_ - 13-April-2017 ------------------------- Added +++++ - - Confirm support for Django 1.11. - - Support for Redshift database URL +- Confirm support for Django 1.11. +- Support for Redshift database URL. Changed +++++++ - - Fix uwsgi settings reload problem (#55) - - Update support for ``django-redis`` urls (#109) +- Fix uwsgi settings reload problem + `#55 `_. +- Update support for ``django-redis`` urls + `#109 `_. `v0.4.1`_ - 13-November-2016 ---------------------------- Added +++++ - - Add support for Django 1.10 +- Add support for Django 1.10. Changed +++++++ - - Fix for unsafe characters into URLs - - Clarifying warning on missing or unreadable file. Thanks to @nickcatal - - Fix support for Oracle urls - - Fix support for ``django-redis`` +- Fix for unsafe characters into URLs. +- Clarifying warning on missing or unreadable file. + Thanks to `@nickcatal `_. +- Fix support for Oracle urls. +- Fix support for ``django-redis``. `v0.4`_ - 23-September-2015 --------------------------- Added +++++ - - New email schemes - ``smtp+ssl`` and ``smtp+tls`` (``smtps`` would be deprecated) - - Add tuple support. Thanks to @anonymouzz - - Add LDAP url support for database. Thanks to ``django-ldapdb`` +- New email schemes - ``smtp+ssl`` and ``smtp+tls`` (``smtps`` would be deprecated). +- Add tuple support. Thanks to `@anonymouzz `_. +- Add LDAP url support for database. Thanks to + `django-ldapdb/django-ldapdb `_. Changed +++++++ - - Fix non-ascii values (broken in Python 2.x) - - ``redis_cache`` replaced by ``django_redis`` - - Fix psql/pgsql url +- Fix non-ascii values (broken in Python 2.x). +- ``redis_cache`` replaced by ``django_redis``. +- Fix psql/pgsql url. `v0.3.1`_ - 19 Sep 2015 ----------------------- Added +++++ - - Added ``email`` as alias for ``email_url`` - - Django 1.7 is now supported - - Added LDAP scheme support for ``db_url_config`` +- Added ``email`` as alias for ``email_url``. +- Django 1.7 is now supported. +- Added LDAP scheme support for ``db_url_config``. Fixed +++++ - - Fixed typos in the documentation - - Fixed ``environ.Path.__add__`` to correctly handle plus operator - - Fixed ``environ.Path.__contains__`` to correctly work on Windows +- Fixed typos in the documentation. +- Fixed ``environ.Path.__add__`` to correctly handle plus operator. +- Fixed ``environ.Path.__contains__`` to correctly work on Windows. `v0.3`_ - 03-June-2014 ---------------------- Added +++++ - - Add cache url support - - Add email url support - - Add search url support +- Add cache url support. +- Add email url support. +- Add search url support. Changed +++++++ - - Rewriting README.rst +- Rewriting README.rst. v0.2.1 - 19-April-2013 ---------------------- Changed +++++++ - - ``Env.__call__`` now uses ``Env.get_value`` instance method +- ``Env.__call__`` now uses ``Env.get_value`` instance method. v0.2 - 16-April-2013 -------------------- Added +++++ - - Add advanced float parsing (comma and dot symbols to separate thousands and decimals) +- Add advanced float parsing (comma and dot symbols to separate thousands and decimals). Fixed +++++ - - Fixed typos in the documentation +- Fixed typos in the documentation. v0.1 - 2-April-2013 ------------------- Added +++++ - - Initial release +- Initial release. .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...develop From 8e049229fa2293968b3b31d59682f4bc748edb0e Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 11:30:41 +0300 Subject: [PATCH 25/71] Add tests for #200 --- tests/test_cache.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/test_cache.py b/tests/test_cache.py index 13077a53..42c72bbc 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -149,3 +149,80 @@ def test_unknown_backend(): def test_empty_url_is_mapped_to_empty_config(): assert Env.cache_url_config('') == {} assert Env.cache_url_config(None) == {} + + +@pytest.mark.parametrize( + 'chars', + ['!', '$', '&', "'", '(', ')', '*', '+', ';', '=', '-', '.', '-v1.2'] +) +def test_cache_url_password_using_sub_delims(monkeypatch, chars): + """Ensure CACHE_URL passwords may contains some unsafe characters. + + See: https://github.com/joke2k/django-environ/issues/200 for details.""" + url = 'rediss://enigma:secret{}@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + result = env.cache_url_config(url) + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + url = 'rediss://enigma:sec{}ret@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + result = env.cache_url_config(url) + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + url = 'rediss://enigma:{}secret@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + result = env.cache_url_config(url) + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + +@pytest.mark.parametrize( + 'chars', ['%3A', '%2F', '%3F', '%23', '%5B', '%5D', '%40', '%2C'] +) +def test_cache_url_password_using_gen_delims(monkeypatch, chars): + """Ensure CACHE_URL passwords may contains %-encoded characters. + + See: https://github.com/joke2k/django-environ/issues/200 for details.""" + url = 'rediss://enigma:secret{}@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + url = 'rediss://enigma:sec{}ret@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url + + url = 'rediss://enigma:{}secret@ondigitalocean.com:25061/2'.format(chars) + monkeypatch.setenv('CACHE_URL', url) + env = Env() + + result = env.cache() + assert result['BACKEND'] == 'django_redis.cache.RedisCache' + assert result['LOCATION'] == url From e928b6aea636ef0a55dcc8e936af5c97a27f6750 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 12:51:18 +0300 Subject: [PATCH 26/71] Safely evaluate a string containing an invalid Python literal See https://github.com/joke2k/django-environ/issues/200 for details --- CHANGELOG.rst | 5 +++-- environ/environ.py | 2 +- tests/test_utils.py | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/test_utils.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e4bd831..335a267d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,12 +17,13 @@ Added `#314 `_. - Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. - Fixed +++++ -- Fixed links in the documentation +- Fixed links in the documentation. - Use default option in ``Env.bytes()`` `#206 `_. +- Safely evaluate a string containing an invalid Python literal + `#200 `_. Changed +++++++ diff --git a/environ/environ.py b/environ/environ.py index 0b97c5e9..0267add6 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -44,7 +44,7 @@ def _cast(value): # https://docs.python.org/3/library/ast.html#ast.literal_eval try: return ast.literal_eval(value) - except ValueError: + except (ValueError, SyntaxError): return value diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..523a72d3 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,22 @@ +# This file is part of the django-environ. +# +# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2013-2021, Daniele Faraglia +# +# For the full copyright and license information, please view +# the LICENSE.txt file that was distributed with this source code. + +import pytest +from environ.environ import _cast + + +@pytest.mark.parametrize( + 'literal', + ['anything-', 'anything*', '*anything', 'anything.', + 'anything.1', '(anything', 'anything-v1.2', 'anything-1.2', 'anything='] +) +def test_cast(literal): + """Safely evaluate a string containing an invalid Python literal. + + See https://github.com/joke2k/django-environ/issues/200 for details.""" + assert _cast(literal) == literal From e04667c4e9a0ec0cbe69a522a0ebb7d4ca4917b0 Mon Sep 17 00:00:00 2001 From: James Lin Date: Tue, 7 Sep 2021 22:45:47 +1200 Subject: [PATCH 27/71] Add the ability to use env variable pointing to a .env file (#242) --- environ/environ.py | 1 + tests/test_env.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index 0267add6..2cac8687 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -748,6 +748,7 @@ def read_env(cls, env_file=None, **overrides): os.path.dirname(frame.f_back.f_code.co_filename), '.env' ) + env_file = os.environ.get('ENV_FILE') or env_file if not os.path.exists(env_file): logger.info( "%s doesn't exist - if you're not configuring your " diff --git a/tests/test_env.py b/tests/test_env.py index efb09fbc..8eacd359 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -351,3 +351,18 @@ class MyEnv(Env): def test_singleton_environ(self): assert self.CONFIG is self.env.ENVIRON + + +class FileEnvByEnvVarTests(TestEnv): + + def setup_method(self, method): + super(FileEnvByEnvVarTests, self).setUp() + with open('./tmp/test_env.txt') as env_file: + env_file.writeline('VAR=TEST') + Env.ENVIRON = {} + os.environ['ENV_FILE'] = './tmp/test_env.txt' + self.env = Env() + + def test_read_env(self): + assert_type_and_value(str, 'TEST', self.env('VAR')) + From 815078d671df887335bb534c25e338e09f6bb253 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 14:05:54 +0300 Subject: [PATCH 28/71] Update documentation --- CHANGELOG.rst | 2 ++ docs/tips.rst | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 335a267d..dc6d1e4f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ Added - Added Elasticsearch7 to search scheme `#314 `_. - Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. +- Added the ability to use env variable pointing to a ``.env`` file + `#242 `_ Fixed +++++ diff --git a/docs/tips.rst b/docs/tips.rst index 20a8b360..0fe21141 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -131,6 +131,7 @@ It is possible to have multiple env files and select one using environment varia Now ``ENV_PATH=other-env ./manage.py runserver`` uses ``other-env`` while ``./manage.py runserver`` uses ``.env``. + Using Path objects when reading env =================================== @@ -154,3 +155,25 @@ It is possible to use of ``pathlib.Path`` objects when reading environment file env.read_env(os.path.join(BASE_DIR, '.env')) env.read_env(pathlib.Path(str(BASE_DIR)).joinpath('.env')) env.read_env(pathlib.Path(str(BASE_DIR)) / '.env') + + +Using an environment variable to point to a .env file +===================================================== + +Since v0.7.0 there is an ability to set an environment variable ``ENV_FILE`` +to point to the .env file location. It can be convenient in a production +systems with a different .env file location. + +The following example demonstrates the above: + +.. code-block:: shell + + # .secret file contents + SOME_VAR=3.14 + +.. code-block:: console + + $ export ENV_FILE=$(pwd)/.secret + $ python -c 'from environ.environ import Env; Env.read_env(); print(Env.ENVIRON["SOME_VAR"])' + 3.14 + From 770e968204ffd349260dddea5e553c044ba4cc67 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 14:13:04 +0300 Subject: [PATCH 29/71] Update doc formatting and change log --- CHANGELOG.rst | 4 ++-- docs/getting-started.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dc6d1e4f..1f4053e6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,9 +15,9 @@ Added `#297 `_. - Added Elasticsearch7 to search scheme `#314 `_. -- Provided ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. +- Added the ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. - Added the ability to use env variable pointing to a ``.env`` file - `#242 `_ + `#242 `_. Fixed +++++ diff --git a/docs/getting-started.rst b/docs/getting-started.rst index c1ae7a25..d44724b9 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -18,7 +18,7 @@ _________________________ ``django-environ`` is a Python-only package `hosted on PyPI `_. The recommended installation method is `pip `_-installing into a virtualenv: -.. code-block:: shell +.. code-block:: console $ python -m pip install django-environ @@ -34,7 +34,7 @@ So, you can also install the latest unreleased development version directly from ``develop`` branch on GitHub. It is a work-in-progress of a future stable release so the experience might be not as smooth.: -.. code-block:: shell +.. code-block:: console $ pip install -e git://github.com/joke2k/django-environ.git#egg=django-environ # OR From e2318913b75c271cef78e98170ac662b65f462fc Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 15:17:44 +0300 Subject: [PATCH 30/71] Revert "Add the ability to use env variable pointing to a .env file (#242)" (#326) This reverts commit e04667c4e9a0ec0cbe69a522a0ebb7d4ca4917b0. --- environ/environ.py | 1 - tests/test_env.py | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 2cac8687..0267add6 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -748,7 +748,6 @@ def read_env(cls, env_file=None, **overrides): os.path.dirname(frame.f_back.f_code.co_filename), '.env' ) - env_file = os.environ.get('ENV_FILE') or env_file if not os.path.exists(env_file): logger.info( "%s doesn't exist - if you're not configuring your " diff --git a/tests/test_env.py b/tests/test_env.py index 8eacd359..efb09fbc 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -351,18 +351,3 @@ class MyEnv(Env): def test_singleton_environ(self): assert self.CONFIG is self.env.ENVIRON - - -class FileEnvByEnvVarTests(TestEnv): - - def setup_method(self, method): - super(FileEnvByEnvVarTests, self).setUp() - with open('./tmp/test_env.txt') as env_file: - env_file.writeline('VAR=TEST') - Env.ENVIRON = {} - os.environ['ENV_FILE'] = './tmp/test_env.txt' - self.env = Env() - - def test_read_env(self): - assert_type_and_value(str, 'TEST', self.env('VAR')) - From 6a67a006440d53535b5591d2898677d23ce951be Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 15:24:25 +0300 Subject: [PATCH 31/71] Update documentation --- CHANGELOG.rst | 2 -- docs/getting-started.rst | 4 +++- docs/tips.rst | 42 +++++++++++++++++----------------------- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1f4053e6..d99fbec5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,8 +16,6 @@ Added - Added Elasticsearch7 to search scheme `#314 `_. - Added the ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. -- Added the ability to use env variable pointing to a ``.env`` file - `#242 `_. Fixed +++++ diff --git a/docs/getting-started.rst b/docs/getting-started.rst index d44724b9..d0d5473f 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -114,7 +114,9 @@ FAQ ``django-environ`` will try to get and read ``.env`` file from the project root if you haven't specified the path for it when call ``read_env``. However, this is not the recommended way. When it is possible always specify - the path tho ``.env`` file. + the path tho ``.env`` file. Alternatively, you can use a trick with a + environment variable pointing to the actual location of .env file. + For details see ``__. #. **What (where) is the root part of the project, is it part of the project where are settings?** diff --git a/docs/tips.rst b/docs/tips.rst index 0fe21141..b743cfa7 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -122,14 +122,30 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to Multiple env files ================== -It is possible to have multiple env files and select one using environment variables. +There is an ability point to the .env file location using an environment +variable. This feature may be convenient in a production systems with a +different .env file location. + +The following example demonstrates the above: + +.. code-block:: shell + + # /etc/environment file contents + DEBUG=False + +.. code-block:: shell + + # .env file contents + DEBUG=True .. code-block:: python env = environ.Env() env.read_env(env.str('ENV_PATH', '.env')) -Now ``ENV_PATH=other-env ./manage.py runserver`` uses ``other-env`` while ``./manage.py runserver`` uses ``.env``. + +Now ``ENV_PATH=/etc/environment ./manage.py runserver`` uses ``/etc/environment`` +while ``./manage.py runserver`` uses ``.env``. Using Path objects when reading env @@ -155,25 +171,3 @@ It is possible to use of ``pathlib.Path`` objects when reading environment file env.read_env(os.path.join(BASE_DIR, '.env')) env.read_env(pathlib.Path(str(BASE_DIR)).joinpath('.env')) env.read_env(pathlib.Path(str(BASE_DIR)) / '.env') - - -Using an environment variable to point to a .env file -===================================================== - -Since v0.7.0 there is an ability to set an environment variable ``ENV_FILE`` -to point to the .env file location. It can be convenient in a production -systems with a different .env file location. - -The following example demonstrates the above: - -.. code-block:: shell - - # .secret file contents - SOME_VAR=3.14 - -.. code-block:: console - - $ export ENV_FILE=$(pwd)/.secret - $ python -c 'from environ.environ import Env; Env.read_env(); print(Env.ENVIRON["SOME_VAR"])' - 3.14 - From ef946bbfc1efb9d48a0a3eb9fe693a215e9f9769 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 7 Sep 2021 17:18:53 +0300 Subject: [PATCH 32/71] Fix documentation issues --- CHANGELOG.rst | 2 +- docs/conf.py | 6 ++++-- docs/getting-started.rst | 2 +- docs/tips.rst | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d99fbec5..0d0179cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added - Added Elasticsearch5 to search scheme `#297 `_. - Added Elasticsearch7 to search scheme - `#314 `_. + `#314 `_. - Added the ability to use ``bytes`` or ``str`` as a default value for ``Env.bytes()``. Fixed diff --git a/docs/conf.py b/docs/conf.py index f11a4d50..17536146 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ def find_version(meta_file): "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.todo", - "notfound.extension" + "notfound.extension", ] # Add any paths that contain templates here, relative to this directory. @@ -66,7 +66,9 @@ def find_version(meta_file): source_suffix = ".rst" # Allow non-local URIs so we can have images in CHANGELOG etc. -suppress_warnings = ["image.nonlocal_uri"] +suppress_warnings = [ + "image.nonlocal_uri", +] # The master toctree document. master_doc = "index" diff --git a/docs/getting-started.rst b/docs/getting-started.rst index d0d5473f..007e92f3 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -116,7 +116,7 @@ FAQ However, this is not the recommended way. When it is possible always specify the path tho ``.env`` file. Alternatively, you can use a trick with a environment variable pointing to the actual location of .env file. - For details see ``__. + For details see ":ref:`multiple-env-files-label`". #. **What (where) is the root part of the project, is it part of the project where are settings?** diff --git a/docs/tips.rst b/docs/tips.rst index b743cfa7..a09391f7 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -119,6 +119,8 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +.. _multiple-env-files-label: + Multiple env files ================== From 1afd39ad3aab052c8b01da6bc1de649edcbda306 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Fri, 10 Sep 2021 00:24:19 +0300 Subject: [PATCH 33/71] Add Acknowledgments section --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index ab55177e..6bf39824 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -6,6 +6,9 @@ and currently maintained by `Serghei Iakovlev `_ A full list of contributors can be found in `GitHub `__. +Acknowledgments +=============== + The existence of ``django-environ`` would have been impossible without these projects: From d0a2b4640d355bf9a2baa1eda96e1b6d95d82751 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 11 Sep 2021 15:31:54 +0300 Subject: [PATCH 34/71] Update change log --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d0179cf..3b3f5576 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.7.0`_ - 00-Unreleased-2021 +`v0.7.0`_ - 11-September-2021 ------------------------------ Added +++++ @@ -219,7 +219,7 @@ Added - Initial release. -.. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...develop +.. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 .. _v0.4.5: https://github.com/joke2k/django-environ/compare/v0.4.4...v0.4.5 From a8871d0199d250a8adefe77109ee2735ece843a5 Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Wed, 15 Sep 2021 19:53:04 +1200 Subject: [PATCH 35/71] Add docker-style file variable support (fixes #189) (#328) --- README.rst | 2 + docs/tips.rst | 37 +++++++ environ/environ.py | 19 ++++ environ/fileaware_mapping.py | 82 ++++++++++++++++ tests/test_fileaware.py | 180 +++++++++++++++++++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 environ/fileaware_mapping.py create mode 100644 tests/test_fileaware.py diff --git a/README.rst b/README.rst index df5a31ca..99a52ef1 100644 --- a/README.rst +++ b/README.rst @@ -113,6 +113,8 @@ a concrete example on using with a django project. - Fill ``os.environ`` with .env file variables - Variables casting - Url variables exploded to django specific package settings +- Optional support for Docker-style file based config variables (use + ``environ.FileAwareEnv`` instead of ``environ.Env``) .. -project-information- diff --git a/docs/tips.rst b/docs/tips.rst index a09391f7..777d5589 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -3,6 +3,42 @@ Tips ==== +Docker-style file based variables +================================= + +Docker (swarm) and Kubernetes are two widely used platforms that store their +secrets in tmpfs inside containers as individual files, providing a secure way +to be able to share configuration data between containers. + +Use ``environ.FileAwareEnv`` rather than ``environ.Env`` to first look for +environment variables with ``_FILE`` appended. If found, their contents will be +read from the file system and used instead. + +For example, given an app with the following in its settings module: + +.. code-block:: python + + import environ + + env = environ.FileAwareEnv() + SECRET_KEY = env("SECRET_KEY") + +the example ``docker-compose.yml`` for would contain: + +.. code-block:: yaml + + secrets: + secret_key: + external: true + + services: + app: + secrets: + - secret_key + environment: + - SECRET_KEY_FILE=/run/secrets/secret_key + + Using unsafe characters in URLs =============================== @@ -14,6 +50,7 @@ In order to use unsafe characters you have to encode with ``urllib.parse.encode` See https://perishablepress.com/stop-using-unsafe-characters-in-urls/ for reference. + Smart Casting ============= diff --git a/environ/environ.py b/environ/environ.py index 0267add6..d312c296 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -27,6 +27,7 @@ ) from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, REDIS_DRIVER +from .fileaware_mapping import FileAwareMapping try: from os import PathLike @@ -786,6 +787,24 @@ def read_env(cls, env_file=None, **overrides): cls.ENVIRON.setdefault(key, value) +class FileAwareEnv(Env): + """ + First look for environment variables with ``_FILE`` appended. If found, + their contents will be read from the file system and used instead. + + Use as a drop-in replacement for the standard ``environ.Env``: + + .. code-block:: python + + python env = environ.FileAwareEnv() + + For example, if a ``SECRET_KEY_FILE`` environment variable was set, + ``env("SECRET_KEY")`` would find the related variable, returning the file + contents rather than ever looking up a ``SECRET_KEY`` environment variable. + """ + ENVIRON = FileAwareMapping() + + class Path: """Inspired to Django Two-scoops, handling File Paths in Settings.""" diff --git a/environ/fileaware_mapping.py b/environ/fileaware_mapping.py new file mode 100644 index 00000000..e26ee8d1 --- /dev/null +++ b/environ/fileaware_mapping.py @@ -0,0 +1,82 @@ +import os +from collections.abc import MutableMapping + + +class FileAwareMapping(MutableMapping): + """ + A mapping that wraps os.environ, first checking for the existance of a key + appended with ``_FILE`` whenever reading a value. If a matching file key is + found then the value is instead read from the file system at this location. + + By default, values read from the file system are cached so future lookups + do not hit the disk again. + + A ``_FILE`` key has higher precidence than a value is set directly in the + environment, and an exception is raised if the file can not be found. + """ + + def __init__(self, env=None, cache=True): + """ + Initialize the mapping. + + :param env: + where to read environment variables from (defaults to + ``os.environ``) + :param cache: + cache environment variables read from the file system (defaults to + ``True``) + """ + self.env = env if env is not None else os.environ + self.cache = cache + self.files_cache = {} + + def __getitem__(self, key): + if self.cache and key in self.files_cache: + return self.files_cache[key] + key_file = self.env.get(key + "_FILE") + if key_file: + with open(key_file) as f: + value = f.read() + if self.cache: + self.files_cache[key] = value + return value + return self.env[key] + + def __iter__(self): + """ + Iterate all keys, also always including the shortened key if ``_FILE`` + keys are found. + """ + for key in self.env: + yield key + if key.endswith("_FILE"): + no_file_key = key[:-5] + if no_file_key and no_file_key not in self.env: + yield no_file_key + + def __len__(self): + """ + Return the length of the file, also always counting shortened keys for + any ``_FILE`` key found. + """ + return len(tuple(iter(self))) + + def __setitem__(self, key, value): + self.env[key] = value + if self.cache and key.endswith("_FILE"): + no_file_key = key[:-5] + if no_file_key and no_file_key in self.files_cache: + del self.files_cache[no_file_key] + + def __delitem__(self, key): + file_key = key + "_FILE" + if file_key in self.env: + del self[file_key] + if key in self.env: + del self.env[key] + return + if self.cache and key.endswith("_FILE"): + no_file_key = key[:-5] + if no_file_key and no_file_key in self.files_cache: + del self.files_cache[no_file_key] + del self.env[key] diff --git a/tests/test_fileaware.py b/tests/test_fileaware.py new file mode 100644 index 00000000..11e9d8a1 --- /dev/null +++ b/tests/test_fileaware.py @@ -0,0 +1,180 @@ +import os +import tempfile +from contextlib import contextmanager + +import environ +import pytest + + +@contextmanager +def make_temp_file(text): + with tempfile.NamedTemporaryFile("w", delete=False) as f: + f.write(text) + f.close() + try: + yield f.name + finally: + if os.path.exists(f.name): + os.unlink(f.name) + + +@pytest.fixture +def tmp_f(): + with make_temp_file(text="fish") as f_name: + yield f_name + + +def test_mapping(tmp_f): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f}) + assert env["ANIMAL"] == "fish" + + +def test_precidence(tmp_f): + env = environ.FileAwareMapping( + env={ + "ANIMAL_FILE": tmp_f, + "ANIMAL": "cat", + } + ) + assert env["ANIMAL"] == "fish" + + +def test_missing_file_raises_exception(): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": "non-existant-file"}) + with pytest.raises(FileNotFoundError): + env["ANIMAL"] + + +def test_iter(): + env = environ.FileAwareMapping( + env={ + "ANIMAL_FILE": "some-file", + "VEGETABLE": "leek", + "VEGETABLE_FILE": "some-vegetable-file", + } + ) + keys = set(env) + assert keys == {"ANIMAL_FILE", "ANIMAL", "VEGETABLE", "VEGETABLE_FILE"} + assert "ANIMAL" in keys + + +def test_len(): + env = environ.FileAwareMapping( + env={ + "ANIMAL_FILE": "some-file", + "VEGETABLE": "leek", + "VEGETABLE_FILE": "some-vegetable-file", + } + ) + assert len(env) == 4 + + +def test_cache(tmp_f): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f}) + assert env["ANIMAL"] == "fish" + + with open(tmp_f, "w") as f: + f.write("cat") + assert env["ANIMAL"] == "fish" + + os.unlink(tmp_f) + assert not os.path.exists(env["ANIMAL_FILE"]) + assert env["ANIMAL"] == "fish" + + +def test_no_cache(tmp_f): + env = environ.FileAwareMapping( + cache=False, + env={"ANIMAL_FILE": tmp_f}, + ) + assert env["ANIMAL"] == "fish" + + with open(tmp_f, "w") as f: + f.write("cat") + assert env["ANIMAL"] == "cat" + + os.unlink(tmp_f) + assert not os.path.exists(env["ANIMAL_FILE"]) + with pytest.raises(FileNotFoundError): + assert env["ANIMAL"] + + +def test_setdefault(tmp_f): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f}) + assert env.setdefault("FRUIT", "apple") == "apple" + assert env.setdefault("ANIMAL", "cat") == "fish" + assert env.env == {"ANIMAL_FILE": tmp_f, "FRUIT": "apple"} + + +class TestDelItem: + def test_del_key(self): + env = environ.FileAwareMapping(env={"FRUIT": "apple"}) + del env["FRUIT"] + with pytest.raises(KeyError): + env["FRUIT"] + + def test_del_key_with_file_key(self): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": "some-file"}) + del env["ANIMAL"] + with pytest.raises(KeyError): + env["ANIMAL"] + + def test_del_shadowed_key_with_file_key(self): + env = environ.FileAwareMapping( + env={"ANIMAL_FILE": "some-file", "ANIMAL": "cat"} + ) + del env["ANIMAL"] + with pytest.raises(KeyError): + env["ANIMAL"] + + def test_del_file_key(self): + env = environ.FileAwareMapping( + env={ + "ANIMAL_FILE": "some-file", + "ANIMAL": "fish", + } + ) + del env["ANIMAL_FILE"] + assert env["ANIMAL"] == "fish" + + def test_del_file_key_clears_cache(self, tmp_f): + env = environ.FileAwareMapping( + env={ + "ANIMAL_FILE": tmp_f, + "ANIMAL": "cat", + } + ) + assert env["ANIMAL"] == "fish" + del env["ANIMAL_FILE"] + assert env["ANIMAL"] == "cat" + + +class TestSetItem: + def test_set_key(self): + env = environ.FileAwareMapping(env={"FRUIT": "apple"}) + env["FRUIT"] = "banana" + assert env["FRUIT"] == "banana" + + def test_cant_override_key_with_file_key(self, tmp_f): + env = environ.FileAwareMapping( + env={ + "FRUIT": "apple", + "FRUIT_FILE": tmp_f, + } + ) + with open(tmp_f, "w") as f: + f.write("banana") + env["FRUIT"] = "cucumber" + assert env["FRUIT"] == "banana" + + def test_set_file_key(self, tmp_f): + env = environ.FileAwareMapping(env={"ANIMAL": "cat"}) + env["ANIMAL_FILE"] = tmp_f + assert env["ANIMAL"] == "fish" + + def test_change_file_key_clears_cache(self, tmp_f): + env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f}) + assert env["ANIMAL"] == "fish" + with make_temp_file(text="cat") as new_tmp_f: + env["ANIMAL_FILE"] = new_tmp_f + assert env["ANIMAL"] == "cat" From 9a30af706e365a2a194062e4d38e48bc0c4869e6 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 11 Sep 2021 18:34:58 +0300 Subject: [PATCH 36/71] Bump version --- CHANGELOG.rst | 5 +++++ environ/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b3f5576..cf3b0003 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.8.0`_ - 00-Unreleased-2021 +------------------------------ + + `v0.7.0`_ - 11-September-2021 ------------------------------ Added @@ -219,6 +223,7 @@ Added - Initial release. +.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...develop .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 diff --git a/environ/__init__.py b/environ/__init__.py index bf8689a0..e37b6e65 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.7.0' +__version__ = '0.8.0' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' From 57cdc98b6d43db3ec908f40b4fcfc1980b97d572 Mon Sep 17 00:00:00 2001 From: John Bergvall Date: Wed, 15 Sep 2021 00:52:10 +0200 Subject: [PATCH 37/71] Keep newline/tab escapes in quoted strings (fix certificate serializing) (#296) Limit backslash escape removal for quoted newlines Keep escape backslash for newline/tab characters. Parsing still differs from shell syntax but keeps string structure intact. Example certificate data VAR="---BEGIN---\r\n---END---" now becomes "---BEGIN---\r\n---END---" instead of "---BEGIN---rn---END---" Co-authored-by: Serghei Iakovlev --- CHANGELOG.rst | 3 +++ docs/tips.rst | 49 +++++++++++++++++++++++++++++++++++++++++----- environ/environ.py | 12 ++++++++++-- tests/fixtures.py | 2 ++ tests/test_env.py | 4 ++++ tests/test_env.txt | 2 ++ 6 files changed, 65 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf3b0003..c067f280 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ and this project adheres to `Semantic Versioning >> print env.str('MULTILINE_TEXT', multiline=True) - Hello - World + # settings.py file contents + import environ + + + env = environ.Env() + + print(env.str('UNQUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('UNQUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('QUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('QUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('ESCAPED_CERT', multiline=True)) + # ---BEGIN---\ + # ---END--- + print(env.str('ESCAPED_CERT', multiline=False)) + # ---BEGIN---\\n---END--- Proxy value =========== diff --git a/environ/environ.py b/environ/environ.py index d312c296..12d0ca1f 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -181,7 +181,7 @@ def str(self, var, default=NOTSET, multiline=False): """ value = self.get_value(var, cast=str, default=default) if multiline: - return value.replace('\\n', '\n') + return re.sub(r'(\\r)?\\n', r'\n', value) return value def unicode(self, var, default=NOTSET): @@ -770,6 +770,13 @@ def read_env(cls, env_file=None, **overrides): logger.debug('Read environment variables from: {}'.format(env_file)) + def _keep_escaped_format_characters(match): + """Keep escaped newline/tabs in quoted strings""" + escaped_char = match.group(1) + if escaped_char in 'rnt': + return '\\' + escaped_char + return escaped_char + for line in content.splitlines(): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: @@ -779,7 +786,8 @@ def read_env(cls, env_file=None, **overrides): val = m2.group(1) m3 = re.match(r'\A"(.*)"\Z', val) if m3: - val = re.sub(r'\\(.)', r'\1', m3.group(1)) + val = re.sub(r'\\(.)', _keep_escaped_format_characters, + m3.group(1)) cls.ENVIRON.setdefault(key, str(val)) # set defaults diff --git a/tests/fixtures.py b/tests/fixtures.py index f685f5cb..29b4dc9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -31,6 +31,8 @@ class FakeEnv: def generate_data(cls): return dict(STR_VAR='bar', MULTILINE_STR_VAR='foo\\nbar', + MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---', + MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---', INT_VAR='42', FLOAT_VAR='33.3', FLOAT_COMMA_VAR='33,3', diff --git a/tests/test_env.py b/tests/test_env.py index efb09fbc..afa6470b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -57,6 +57,10 @@ def test_contains(self): ('STR_VAR', 'bar', False), ('MULTILINE_STR_VAR', 'foo\\nbar', False), ('MULTILINE_STR_VAR', 'foo\nbar', True), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\\r\\n---END---', False), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\n---END---', True), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\\n---END---', False), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\n---END---', True), ], ) def test_str(self, var, val, multiline): diff --git a/tests/test_env.txt b/tests/test_env.txt index dfbd61ef..befeed76 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -33,6 +33,8 @@ INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar MULTILINE_STR_VAR=foo\nbar +MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---" +MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END--- INT_LIST=42,33 CYRILLIC_VAR=фуубар INT_TUPLE=(42,33) From d3e25b9cf8e17cee65771ceaf5686d1157a775b2 Mon Sep 17 00:00:00 2001 From: Eduardo Lima Date: Mon, 4 Jan 2021 16:33:44 -0300 Subject: [PATCH 38/71] Log invalid lines --- environ/environ.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index 12d0ca1f..83642ff5 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -789,6 +789,8 @@ def _keep_escaped_format_characters(match): val = re.sub(r'\\(.)', _keep_escaped_format_characters, m3.group(1)) cls.ENVIRON.setdefault(key, str(val)) + else: + logger.warn('Invalid line: %s', line) # set defaults for key, value in overrides.items(): From e99045386d8744f2428c5f34dda602ee2a3967f0 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 02:17:12 +0300 Subject: [PATCH 39/71] Update change log --- CHANGELOG.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c067f280..28e97cbb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,15 @@ and this project adheres to `Semantic Versioning `_. + Fixed +++++ -- Keep newline/tb escaped in quoted strings +- Keep newline/tab escapes in quoted strings + `#296 `_. `v0.7.0`_ - 11-September-2021 From e2c1412987c4250f9ee4a08f82a7bc91dc14de78 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 02:17:52 +0300 Subject: [PATCH 40/71] Update change log --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 28e97cbb..07cc1dbe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ Added Fixed +++++ - Keep newline/tab escapes in quoted strings - `#296 `_. + `#296 `_. `v0.7.0`_ - 11-September-2021 From cbbd6d6d454352e4ff712947502466a84dd8faef Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 15 Sep 2021 11:00:24 +0300 Subject: [PATCH 41/71] Update change log --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 07cc1dbe..b64dc00c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Added +++++ - Log invalid lines when parse .env file `#283 `_. +- Added docker-style file variable support + `#189 `_. Fixed +++++ From 790106fbe4348c1a19e980f543a373e29a3e8319 Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 28 Sep 2021 03:38:36 +1300 Subject: [PATCH 42/71] Option to override existing variables with read_env (#329) * Option to override existing variables with read_env Fixes #249 * Improve Python 3.5 support * Add another test to bump up coverage since my simplified code dropped the percentage :P --- docs/tips.rst | 20 ++++++++++++++++++-- environ/environ.py | 25 +++++++++++++++++-------- tests/test_env.py | 33 ++++++++++++++++++++++----------- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/docs/tips.rst b/docs/tips.rst index ab3161dd..cff0d862 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -195,10 +195,13 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +Reading env files +================= + .. _multiple-env-files-label: Multiple env files -================== +------------------ There is an ability point to the .env file location using an environment variable. This feature may be convenient in a production systems with a @@ -227,7 +230,7 @@ while ``./manage.py runserver`` uses ``.env``. Using Path objects when reading env -=================================== +----------------------------------- It is possible to use of ``pathlib.Path`` objects when reading environment file from the filesystem: @@ -249,3 +252,16 @@ It is possible to use of ``pathlib.Path`` objects when reading environment file env.read_env(os.path.join(BASE_DIR, '.env')) env.read_env(pathlib.Path(str(BASE_DIR)).joinpath('.env')) env.read_env(pathlib.Path(str(BASE_DIR)) / '.env') + + +Overwriting existing environment values from env files +------------------------------------------------------ + +If you want variables set within your env files to take higher precidence than +an existing set environment variable, use the ``overwrite=True`` argument of +``read_env``. For example: + +.. code-block:: python + + env = environ.Env() + env.read_env(BASE_DIR('.env'), overwrite=True) diff --git a/environ/environ.py b/environ/environ.py index 83642ff5..a38324f4 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -31,10 +31,10 @@ try: from os import PathLike +except ImportError: # Python 3.5 support + from pathlib import PurePath as PathLike - Openable = (str, PathLike) -except ImportError: - Openable = (str,) +Openable = (str, PathLike) logger = logging.getLogger(__name__) @@ -732,13 +732,16 @@ def search_url_config(cls, url, engine=None): return config @classmethod - def read_env(cls, env_file=None, **overrides): + def read_env(cls, env_file=None, overwrite=False, **overrides): """Read a .env file into os.environ. If not given a path to a dotenv path, does filthy magic stack backtracking to find the dotenv in the same directory as the file that called read_env. + By default, won't overwrite any existing environment variables. You can + enable this behaviour by setting ``overwrite=True``. + Refs: - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 @@ -757,7 +760,8 @@ def read_env(cls, env_file=None, **overrides): try: if isinstance(env_file, Openable): - with open(env_file) as f: + # Python 3.5 support (wrap path with str). + with open(str(env_file)) as f: content = f.read() else: with env_file as f: @@ -788,13 +792,18 @@ def _keep_escaped_format_characters(match): if m3: val = re.sub(r'\\(.)', _keep_escaped_format_characters, m3.group(1)) - cls.ENVIRON.setdefault(key, str(val)) + overrides[key] = str(val) else: logger.warn('Invalid line: %s', line) - # set defaults + if overwrite: + def set_environ(key, value): + cls.ENVIRON[key] = value + else: + set_environ = cls.ENVIRON.setdefault + for key, value in overrides.items(): - cls.ENVIRON.setdefault(key, value) + set_environ(key, value) class FileAwareEnv(Env): diff --git a/tests/test_env.py b/tests/test_env.py index afa6470b..acd8dc40 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -152,6 +152,7 @@ def test_dict_value(self): [ ('a=1', dict, {'a': '1'}), ('a=1', dict(value=int), {'a': 1}), + ('a=1', dict(value=float), {'a': 1.0}), ('a=1,2,3', dict(value=[str]), {'a': ['1', '2', '3']}), ('a=1,2,3', dict(value=[int]), {'a': [1, 2, 3]}), ('a=1;b=1.1,2.2;c=3', dict(value=int, cast=dict(b=[float])), @@ -163,6 +164,7 @@ def test_dict_value(self): ids=[ 'dict', 'dict_int', + 'dict_float', 'dict_str_list', 'dict_int_list', 'dict_int_cast', @@ -307,34 +309,43 @@ def setup_method(self, method): PATH_VAR=Path(__file__, is_file=True).__root__ ) - def test_read_env_path_like(self): + def create_temp_env_file(self, name): import pathlib import tempfile - path_like = (pathlib.Path(tempfile.gettempdir()) / 'test_pathlib.env') + env_file_path = (pathlib.Path(tempfile.gettempdir()) / name) try: - path_like.unlink() + env_file_path.unlink() except FileNotFoundError: pass - assert not path_like.exists() + assert not env_file_path.exists() + return env_file_path + + def test_read_env_path_like(self): + env_file_path = self.create_temp_env_file('test_pathlib.env') env_key = 'SECRET' env_val = 'enigma' env_str = env_key + '=' + env_val # open() doesn't take path-like on Python < 3.6 - try: - with open(path_like, 'w', encoding='utf-8') as f: - f.write(env_str + '\n') - except TypeError: - return + with open(str(env_file_path), 'w', encoding='utf-8') as f: + f.write(env_str + '\n') - assert path_like.exists() - self.env.read_env(path_like) + self.env.read_env(env_file_path) assert env_key in self.env.ENVIRON assert self.env.ENVIRON[env_key] == env_val + @pytest.mark.parametrize("overwrite", [True, False]) + def test_existing_overwrite(self, overwrite): + env_file_path = self.create_temp_env_file('test_existing.env') + with open(str(env_file_path), 'w') as f: + f.write("EXISTING=b") + self.env.ENVIRON['EXISTING'] = "a" + self.env.read_env(env_file_path, overwrite=overwrite) + assert self.env.ENVIRON["EXISTING"] == ("b" if overwrite else "a") + class TestSubClass(TestEnv): def setup_method(self, method): From fb6a325e9b746919961b79fa82dc9597d158495a Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:01:55 +0300 Subject: [PATCH 43/71] Refactor a bit read_env to simplify overriding env vars --- environ/environ.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index a38324f4..8e9cef37 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -739,12 +739,22 @@ def read_env(cls, env_file=None, overwrite=False, **overrides): backtracking to find the dotenv in the same directory as the file that called read_env. - By default, won't overwrite any existing environment variables. You can - enable this behaviour by setting ``overwrite=True``. + Existing environment variables take precedent and are NOT overwritten + by the file content. ``overwrite=True`` will force an overwrite of + existing environment variables. Refs: - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 + + :param env_file: The path to the `.env` file your application should + use. If a path is not provided, `read_env` will attempt to import + the Django settings module from the Django project root. + :param overwrite: ``overwrite=True`` will force an overwrite of + existing environment variables. + :param **overrides: Any additional keyword arguments provided directly + to read_env will be added to the environment. If the key matches an + existing environment variable, the value will be overridden. """ if env_file is None: frame = sys._getframe() @@ -796,14 +806,19 @@ def _keep_escaped_format_characters(match): else: logger.warn('Invalid line: %s', line) - if overwrite: - def set_environ(key, value): - cls.ENVIRON[key] = value - else: - set_environ = cls.ENVIRON.setdefault + def set_environ(envval): + """Return lambda to set environ. + + Use setdefault unless overwrite is specified. + """ + if overwrite: + return lambda k,v: envval.update({k: str(v)}) + return lambda k,v: envval.setdefault(k,str(v)) + + setenv = set_environ(cls.ENVIRON) for key, value in overrides.items(): - set_environ(key, value) + setenv(key, value) class FileAwareEnv(Env): From 5e337533d585637554ae42e11acc214701e84430 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:02:21 +0300 Subject: [PATCH 44/71] Replace deprecated logger.warn by logger.warning --- environ/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 8e9cef37..42fbffc8 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -804,7 +804,7 @@ def _keep_escaped_format_characters(match): m3.group(1)) overrides[key] = str(val) else: - logger.warn('Invalid line: %s', line) + logger.warning('Invalid line: %s', line) def set_environ(envval): """Return lambda to set environ. From a8718082754246e32d0611655a4f6f487a34ddb7 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:02:36 +0300 Subject: [PATCH 45/71] Add missed copyright notice --- tests/test_fileaware.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_fileaware.py b/tests/test_fileaware.py index 11e9d8a1..b4733bc2 100644 --- a/tests/test_fileaware.py +++ b/tests/test_fileaware.py @@ -1,3 +1,11 @@ +# This file is part of the django-environ. +# +# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2013-2021, Daniele Faraglia +# +# For the full copyright and license information, please view +# the LICENSE.txt file that was distributed with this source code. + import os import tempfile from contextlib import contextmanager From cc0138b673616a62fd87d5d64242712c82ae7a4a Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:04:58 +0300 Subject: [PATCH 46/71] Update change log --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b64dc00c..47d9673f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ Added `#283 `_. - Added docker-style file variable support `#189 `_. +- Added option to override existing variables with ``read_env`` + `#103 `_, + `#249 `_. + Fixed +++++ From 444c3ca8289676b78d4db84d295c7b773d6ccc88 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 27 Sep 2021 18:07:14 +0300 Subject: [PATCH 47/71] Fix code style issues --- environ/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 42fbffc8..b678096d 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -812,8 +812,8 @@ def set_environ(envval): Use setdefault unless overwrite is specified. """ if overwrite: - return lambda k,v: envval.update({k: str(v)}) - return lambda k,v: envval.setdefault(k,str(v)) + return lambda k, v: envval.update({k: str(v)}) + return lambda k, v: envval.setdefault(k, str(v)) setenv = set_environ(cls.ENVIRON) From d482353a932bae0f74c6fc87372a78c604ad6326 Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Tue, 5 Oct 2021 22:26:33 +0330 Subject: [PATCH 48/71] Add test to ensure default value for cache works Signed-off-by: Mehdy Khoshnoody --- tests/test_cache.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_cache.py b/tests/test_cache.py index 42c72bbc..dd4e4626 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -226,3 +226,11 @@ def test_cache_url_password_using_gen_delims(monkeypatch, chars): result = env.cache() assert result['BACKEND'] == 'django_redis.cache.RedisCache' assert result['LOCATION'] == url + + +def test_cache_url_env_using_default(): + env = Env(CACHE_URL=(str, "locmemcache://")) + result = env.cache() + + assert result["BACKEND"] == "django.core.cache.backends.locmem.LocMemCache" + assert result["LOCATION"] == "" From a6d90fb570f59fff65fa5c0a9c72c29e6f39df6b Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Tue, 5 Oct 2021 22:50:47 +0330 Subject: [PATCH 49/71] Add support for empty var with None default value Signed-off-by: Mehdy Khoshnoody --- environ/environ.py | 2 ++ tests/test_env.py | 1 + tests/test_env.txt | 1 + 3 files changed, 4 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index b678096d..7cb20404 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -375,6 +375,8 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): not isinstance(default, NoValue): cast = type(default) + value = None if default is None and value == '' else value + if value != default or (parse_default and value): value = self.parse_value(value, cast) diff --git a/tests/test_env.py b/tests/test_env.py index acd8dc40..55147fbc 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -86,6 +86,7 @@ def test_int(self): def test_int_with_none_default(self): assert self.env('NOT_PRESENT_VAR', cast=int, default=None) is None + assert self.env('EMPTY_INT_VAR', cast=int, default=None) is None @pytest.mark.parametrize( 'value,variable', diff --git a/tests/test_env.txt b/tests/test_env.txt index befeed76..a179a48e 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -29,6 +29,7 @@ FLOAT_STRANGE_VAR2=123.420.333,3 FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR EMPTY_LIST= +EMPTY_INT_VAR= INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar From 26fb010ab301da30158e847c2de159cb73c22dd2 Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Wed, 6 Oct 2021 13:59:01 +0330 Subject: [PATCH 50/71] Add update to CHANGELOG.rst Signed-off-by: Mehdy Khoshnoody --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47d9673f..c46d3ffb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ Added - Added option to override existing variables with ``read_env`` `#103 `_, `#249 `_. +- Added support for empty var with None default value + `#209 `_. Fixed From af4c68db08e2ca28801ead5ab70fcae1181d832e Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Wed, 6 Oct 2021 17:02:40 +0330 Subject: [PATCH 51/71] Handle escaped dollar sign in values Signed-off-by: Mehdy Khoshnoody --- CHANGELOG.rst | 2 ++ environ/environ.py | 5 +++++ tests/fixtures.py | 1 + tests/test_env.py | 8 ++++++++ tests/test_env.txt | 1 + 5 files changed, 17 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47d9673f..b41318e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,8 @@ Fixed +++++ - Keep newline/tab escapes in quoted strings `#296 `_. +- Handle escaped dolloar sign in values + `#271 `_. `v0.7.0`_ - 11-September-2021 diff --git a/environ/environ.py b/environ/environ.py index b678096d..b177784e 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -160,6 +160,7 @@ class Env: def __init__(self, **scheme): self.smart_cast = True + self.escape_proxy = False self.scheme = scheme def __call__(self, var, cast=None, default=NOTSET, parse_default=False): @@ -365,10 +366,14 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' + escape = rb'\$' if isinstance(value, bytes) else r'\$' if hasattr(value, 'startswith') and value.startswith(prefix): value = value.lstrip(prefix) value = self.get_value(value, cast=cast, default=default) + if self.escape_proxy and hasattr(value, 'replace'): + value = value.replace(escape, prefix) + # Smart casting if self.smart_cast: if cast is None and default is not None and \ diff --git a/tests/fixtures.py b/tests/fixtures.py index 29b4dc9f..782123df 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -53,6 +53,7 @@ def generate_data(cls): BOOL_FALSE_STRING_LIKE_BOOL='False', BOOL_FALSE_BOOL=False, PROXIED_VAR='$STR_VAR', + ESCAPED_VAR=r'\$baz', INT_LIST='42,33', INT_TUPLE='(42,33)', STR_LIST_WITH_SPACES=' foo, bar', diff --git a/tests/test_env.py b/tests/test_env.py index acd8dc40..e66cf99d 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -126,6 +126,14 @@ def test_bool_true(self, value, variable): def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' + def test_escaped_dollar_sign(self): + self.env.escape_proxy = True + assert self.env('ESCAPED_VAR') == '$baz' + + def test_escaped_dollar_sign_disabled(self): + self.env.escape_proxy = False + assert self.env('ESCAPED_VAR') == r'\$baz' + def test_int_list(self): assert_type_and_value(list, [42, 33], self.env('INT_LIST', cast=[int])) assert_type_and_value(list, [42, 33], self.env.list('INT_LIST', int)) diff --git a/tests/test_env.txt b/tests/test_env.txt index befeed76..fa593d27 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -28,6 +28,7 @@ FLOAT_STRANGE_VAR1=123,420,333.3 FLOAT_STRANGE_VAR2=123.420.333,3 FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR +ESCAPED_VAR=\$baz EMPTY_LIST= INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar From 1b6c4ba7e1fc1548a5b9692c923a6523a11ef4af Mon Sep 17 00:00:00 2001 From: Mehdy Khoshnoody Date: Thu, 7 Oct 2021 17:12:49 +0330 Subject: [PATCH 52/71] Add docs for escape proxy feature Signed-off-by: Mehdy Khoshnoody --- docs/tips.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/tips.rst b/docs/tips.rst index cff0d862..dbb762fb 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -195,6 +195,23 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +Escape Proxy +============ + +If you're having trouble with values starting with dollar sign ($) without the intention of proxying the value to +another, You should enbale the ``escape_proxy`` and prepend a backslash to it. + +.. code-block:: python + + import environ + + env = environ.Env() + env.escape_proxy = True + + # ESCAPED_VAR=\$baz + env.str('ESCAPED_VAR') # $baz + + Reading env files ================= From f2163279c915b3ff678592bff15373bc13d4881e Mon Sep 17 00:00:00 2001 From: Mirco Grillo Date: Fri, 8 Oct 2021 14:42:40 +0200 Subject: [PATCH 53/71] Fix incorrect parsing when using CloudSQL db url Add test for covering it --- environ/environ.py | 6 +++++- tests/fixtures.py | 2 ++ tests/test_env.py | 3 +++ tests/test_env.txt | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 1ebbb8d7..676d04b4 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -157,6 +157,7 @@ class Env: "xapian": "haystack.backends.xapian_backend.XapianEngine", "simple": "haystack.backends.simple_backend.SimpleEngine", } + CLOUDSQL = 'cloudsql' def __init__(self, **scheme): self.smart_cast = True @@ -502,7 +503,10 @@ def db_url_config(cls, url, engine=None): 'PORT': _cast_int(url.port) or '', }) - if url.scheme in cls.POSTGRES_FAMILY and path.startswith('/'): + if ( + url.scheme in cls.POSTGRES_FAMILY and path.startswith('/') + or cls.CLOUDSQL in path and path.startswith('/') + ): config['HOST'], config['NAME'] = path.rsplit('/', 1) if url.scheme == 'oracle' and path == '': diff --git a/tests/fixtures.py b/tests/fixtures.py index 782123df..25213ac7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -13,6 +13,7 @@ class FakeEnv: URL = 'http://www.google.com/' POSTGRES = 'postgres://uf07k1:wegauwhg@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722' MYSQL = 'mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true' + MYSQL_CLOUDSQL_URL = 'mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase' MYSQLGIS = 'mysqlgis://user:password@127.0.0.1/some_database' SQLITE = 'sqlite:////full/path/to/your/database/file.sqlite' ORACLE_TNS = 'oracle://user:password@sid/' @@ -67,6 +68,7 @@ def generate_data(cls): DATABASE_ORACLE_TNS_URL=cls.ORACLE_TNS, DATABASE_REDSHIFT_URL=cls.REDSHIFT, DATABASE_CUSTOM_BACKEND_URL=cls.CUSTOM_BACKEND, + DATABASE_MYSQL_CLOUDSQL_URL=cls.MYSQL_CLOUDSQL_URL, CACHE_URL=cls.MEMCACHE, CACHE_REDIS=cls.REDIS, EMAIL_URL=cls.EMAIL, diff --git a/tests/test_env.py b/tests/test_env.py index 81aa6545..0be9a60a 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -218,6 +218,8 @@ def test_url_encoded_parts(self): '/full/path/to/your/database/file.sqlite', '', '', '', ''), ('DATABASE_CUSTOM_BACKEND_URL', 'custom.backend', 'database', 'example.com', 'user', 'password', 5430), + ('DATABASE_MYSQL_CLOUDSQL_URL', 'django.db.backends.mysql', 'mydatabase', + '/cloudsql/arvore-codelab:us-central1:mysqlinstance', 'djuser', 'hidden-password', ''), ], ids=[ 'postgres', @@ -228,6 +230,7 @@ def test_url_encoded_parts(self): 'redshift', 'sqlite', 'custom', + 'cloudsql', ], ) def test_db_url_value(self, var, engine, name, host, user, passwd, port): diff --git a/tests/test_env.txt b/tests/test_env.txt index 60ea78c3..c6363ed3 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -1,5 +1,6 @@ DICT_VAR=foo=bar,test=on DATABASE_MYSQL_URL=mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true +DATABASE_MYSQL_CLOUDSQL_URL=mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase DATABASE_MYSQL_GIS_URL=mysqlgis://user:password@127.0.0.1/some_database CACHE_URL=memcache://127.0.0.1:11211 CACHE_REDIS=rediscache://127.0.0.1:6379/1?client_class=django_redis.client.DefaultClient&password=secret From f5ab1f13defd536fe9a80c7e4a777254b12f5f7d Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sat, 9 Oct 2021 10:44:21 +0300 Subject: [PATCH 54/71] Update change log --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50109b18..9e11ac4e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,8 +24,10 @@ Fixed +++++ - Keep newline/tab escapes in quoted strings `#296 `_. -- Handle escaped dolloar sign in values +- Handle escaped dollar sign in values `#271 `_. +- Fixed incorrect parsing of ``DATABASES_URL`` for Google Cloud MySQL + `#294 `_. `v0.7.0`_ - 11-September-2021 From 132e3861ed640610e62f159bca46ecff15892fe1 Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 12 Oct 2021 13:30:16 +1300 Subject: [PATCH 55/71] Add pymemcache cache backend for Django 3.2+ --- docs/types.rst | 6 ++++-- environ/compat.py | 20 ++++++++++++++++---- environ/environ.py | 5 +++-- tests/test_cache.py | 22 ++++++++++++++++++++-- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/types.rst b/docs/types.rst index 1e80e582..292ed64f 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -34,8 +34,10 @@ Supported types * Dummy: ``dummycache://`` * File: ``filecache://`` * Memory: ``locmemcache://`` - * Memcached: ``memcache://`` - * Python memory: ``pymemcache://`` + * Memcached: + * ``memcache://`` (uses ``python-memcached`` backend, deprecated in Django 3.2) + * ``pymemcache://`` (uses ``pymemcache`` backend if Django >=3.2 and package is installed, otherwise will use ``pylibmc`` backend to keep config backwards compatibility) + * ``pylibmc://`` * Redis: ``rediscache://``, ``redis://``, or ``rediss://`` * ``search_url`` diff --git a/environ/compat.py b/environ/compat.py index 0f9eb9a5..9f50f68f 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -8,15 +8,15 @@ """This module handles import compatibility issues.""" -import pkgutil +from pkgutil import find_loader -if pkgutil.find_loader('simplejson'): +if find_loader('simplejson'): import simplejson as json else: import json -if pkgutil.find_loader('django'): +if find_loader('django'): from django import VERSION as DJANGO_VERSION from django.core.exceptions import ImproperlyConfigured else: @@ -33,7 +33,19 @@ class ImproperlyConfigured(Exception): DJANGO_POSTGRES = 'django.db.backends.postgresql' # back compatibility with redis_cache package -if pkgutil.find_loader('redis_cache'): +if find_loader('redis_cache'): REDIS_DRIVER = 'redis_cache.RedisCache' else: REDIS_DRIVER = 'django_redis.cache.RedisCache' + + +# back compatibility for pymemcache +def choose_pymemcache_driver(): + if (DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2)) or not find_loader('pymemcache'): + # The original backend choice for the 'pymemcache' scheme is unfortunately + # 'pylibmc'. + return 'django.core.cache.backends.memcached.PyLibMCCache' + return 'django.core.cache.backends.memcached.PyMemcacheCache' + + +PYMEMCACHE_DRIVER = choose_pymemcache_driver() \ No newline at end of file diff --git a/environ/environ.py b/environ/environ.py index 676d04b4..8b08dc5a 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -26,7 +26,7 @@ urlunparse, ) -from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, REDIS_DRIVER +from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, PYMEMCACHE_DRIVER, REDIS_DRIVER from .fileaware_mapping import FileAwareMapping try: @@ -116,7 +116,8 @@ class Env: 'filecache': 'django.core.cache.backends.filebased.FileBasedCache', 'locmemcache': 'django.core.cache.backends.locmem.LocMemCache', 'memcache': 'django.core.cache.backends.memcached.MemcachedCache', - 'pymemcache': 'django.core.cache.backends.memcached.PyLibMCCache', + 'pymemcache': PYMEMCACHE_DRIVER, + 'pylibmc': 'django.core.cache.backends.memcached.PyLibMCCache', 'rediscache': REDIS_DRIVER, 'redis': REDIS_DRIVER, 'rediss': REDIS_DRIVER, diff --git a/tests/test_cache.py b/tests/test_cache.py index 42c72bbc..53f42110 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -6,10 +6,13 @@ # For the full copyright and license information, please view # the LICENSE.txt file that was distributed with this source code. +from unittest import mock + import pytest +import environ.compat from environ import Env -from environ.compat import REDIS_DRIVER, ImproperlyConfigured +from environ.compat import PYMEMCACHE_DRIVER, REDIS_DRIVER, ImproperlyConfigured def test_base_options_parsing(): @@ -63,7 +66,7 @@ def test_base_options_parsing(): 'django.core.cache.backends.memcached.MemcachedCache', '127.0.0.1:11211'), ('pymemcache://127.0.0.1:11211', - 'django.core.cache.backends.memcached.PyLibMCCache', + PYMEMCACHE_DRIVER, '127.0.0.1:11211'), ], ids=[ @@ -90,6 +93,21 @@ def test_cache_parsing(url, backend, location): assert url['LOCATION'] == location +@pytest.mark.parametrize('django_version', ((3, 2), (3, 1), None)) +@pytest.mark.parametrize('pymemcache_installed', (True, False)) +def test_pymemcache_compat(django_version, pymemcache_installed): + old = 'django.core.cache.backends.memcached.PyLibMCCache' + new = 'django.core.cache.backends.memcached.PyMemcacheCache' + with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): + with mock.patch('environ.compat.find_loader') as mock_find_loader: + mock_find_loader.return_value = pymemcache_installed + driver = environ.compat.choose_pymemcache_driver() + if django_version and django_version < (3, 2): + assert driver == old + else: + assert driver == new if pymemcache_installed else old + + def test_redis_parsing(): url = ('rediscache://127.0.0.1:6379/1?client_class=' 'django_redis.client.DefaultClient&password=secret') From 1c62ff53c91bf610589ba2a8e7d79a889898fc8f Mon Sep 17 00:00:00 2001 From: Chris Beaven Date: Tue, 12 Oct 2021 13:39:01 +1300 Subject: [PATCH 56/71] Lint fixes --- environ/compat.py | 9 +++++---- environ/environ.py | 8 +++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/environ/compat.py b/environ/compat.py index 9f50f68f..8c259f85 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -41,11 +41,12 @@ class ImproperlyConfigured(Exception): # back compatibility for pymemcache def choose_pymemcache_driver(): - if (DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2)) or not find_loader('pymemcache'): - # The original backend choice for the 'pymemcache' scheme is unfortunately - # 'pylibmc'. + old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2) + if old_django or not find_loader('pymemcache'): + # The original backend choice for the 'pymemcache' scheme is + # unfortunately 'pylibmc'. return 'django.core.cache.backends.memcached.PyLibMCCache' return 'django.core.cache.backends.memcached.PyMemcacheCache' -PYMEMCACHE_DRIVER = choose_pymemcache_driver() \ No newline at end of file +PYMEMCACHE_DRIVER = choose_pymemcache_driver() diff --git a/environ/environ.py b/environ/environ.py index 8b08dc5a..505577fc 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -26,7 +26,13 @@ urlunparse, ) -from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, PYMEMCACHE_DRIVER, REDIS_DRIVER +from .compat import ( + DJANGO_POSTGRES, + ImproperlyConfigured, + json, + PYMEMCACHE_DRIVER, + REDIS_DRIVER, +) from .fileaware_mapping import FileAwareMapping try: From 2aa4335972286b136eabb286400c168dd9d9d485 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Thu, 14 Oct 2021 01:38:41 +0300 Subject: [PATCH 57/71] Update change log --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e11ac4e..6f615e45 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,8 @@ Added `#249 `_. - Added support for empty var with None default value `#209 `_. +- Added ``pymemcache`` cache backend for Django 3.2+ + `#335 `_. Fixed From ac3ce78df358ce75480e12f0be859502e67f39a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Oct 2021 17:16:53 +0000 Subject: [PATCH 58/71] Bump actions/checkout from 2.3.4 to 2.3.5 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.4 to 2.3.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.4...v2.3.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f485213..fe711895 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 - name: Set up Python 3.9 uses: actions/setup-python@v2.2.2 @@ -67,7 +67,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 - name: Set up Python 3.9 uses: actions/setup-python@v2.2.2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11f3d39e..c836edc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: fetch-depth: 5 diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index e1e721ca..e4283d10 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 - name: Set up Python 3.9 uses: actions/setup-python@v2.2.2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2c31930a..5c68593c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 - name: Set up Python 3.8 uses: actions/setup-python@v2.2.2 From 813d1038f0ba6b8e2734490fc5d4200d33b2bf30 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 17 Oct 2021 15:22:21 +0300 Subject: [PATCH 59/71] Update change log --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6f615e45..0637b00b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.8.0`_ - 00-Unreleased-2021 +`v0.8.0`_ - 17-October-2021 ------------------------------ Added +++++ @@ -246,7 +246,7 @@ Added - Initial release. -.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...develop +.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 From 96b5804121edae734c313875ec414356def4f9af Mon Sep 17 00:00:00 2001 From: Pratyush Mittal Date: Mon, 18 Oct 2021 16:53:21 +0530 Subject: [PATCH 60/71] Support line breaks and comments in .env file The https://github.com/joke2k/django-environ/pull/283/files commit added useful warnings for skipped lines. However, it also started showing warnings for comment lines. Like in `.zshrc`, we add comments using hash (#). We also group common blocks together. Our env file was something like: ```env SECRET_KEY="S0M3#SecreT" DATABASE_URL=mysql://py-user:p@@sWord@127.0.0.1:3306/db_name GOOGLE_CLIENT_ID="" GOOGLE_CLIENT_SECRET="" ``` --- environ/environ.py | 3 +++ tests/test_env.txt | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/environ/environ.py b/environ/environ.py index 505577fc..42ebcd71 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -821,6 +821,9 @@ def _keep_escaped_format_characters(match): val = re.sub(r'\\(.)', _keep_escaped_format_characters, m3.group(1)) overrides[key] = str(val) + elif not line or line.startswith('#'): + # ignore warnings for empty line-breaks or comments + pass else: logger.warning('Invalid line: %s', line) diff --git a/tests/test_env.txt b/tests/test_env.txt index c6363ed3..78422b18 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -1,10 +1,18 @@ DICT_VAR=foo=bar,test=on + +# Database variables DATABASE_MYSQL_URL=mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true DATABASE_MYSQL_CLOUDSQL_URL=mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase DATABASE_MYSQL_GIS_URL=mysqlgis://user:password@127.0.0.1/some_database + +# Cache variables CACHE_URL=memcache://127.0.0.1:11211 CACHE_REDIS=rediscache://127.0.0.1:6379/1?client_class=django_redis.client.DefaultClient&password=secret + +# Email variables EMAIL_URL=smtps://user@domain.com:password@smtp.example.com:587 + +# Others URL_VAR=http://www.google.com/ PATH_VAR=/home/dev BOOL_TRUE_STRING_LIKE_INT='1' @@ -45,4 +53,6 @@ DATABASE_ORACLE_TNS_URL=oracle://user:password@sid DATABASE_ORACLE_URL=oracle://user:password@host:1521/sid DATABASE_REDSHIFT_URL=redshift://user:password@examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439/dev DATABASE_CUSTOM_BACKEND_URL=custom.backend://user:password@example.com:5430/database + +# Exports export EXPORTED_VAR="exported var" From 5f281492f69f7fa563d0b7df96168c8cf767eac3 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 19 Oct 2021 23:42:09 +0300 Subject: [PATCH 61/71] Bump version --- environ/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/__init__.py b/environ/__init__.py index e37b6e65..97733f4d 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.8.0' +__version__ = '0.8.1' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' From 7361c91b9f415a6fab3f55c534dda1b466dab896 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Tue, 19 Oct 2021 23:42:53 +0300 Subject: [PATCH 62/71] Update change log --- CHANGELOG.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0637b00b..f8c8a3ed 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.8.1`_ - 20-October-2021 + +Fixed ++++++ +- Fixed "Invalid line" spam logs on blank lines in env file + `#340 `_. + + `v0.8.0`_ - 17-October-2021 ------------------------------ Added @@ -246,6 +254,7 @@ Added - Initial release. +.. _v0.8.1: https://github.com/joke2k/django-environ/compare/v0.8.0...v0.8.1 .. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 From 87c928759b7133b3bb93f744072968aa9e8cbd5f Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 00:02:25 +0300 Subject: [PATCH 63/71] Fixed memcache/pymemcache URL parsing. For more see: https://github.com/joke2k/django-environ/issues/337 --- CHANGELOG.rst | 2 ++ environ/environ.py | 10 +++++++++- tests/test_cache.py | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f8c8a3ed..dd34e763 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Fixed +++++ - Fixed "Invalid line" spam logs on blank lines in env file `#340 `_. +- Fixed ``memcache``/``pymemcache`` URL parsing for correct identification of + connection type `#337 `_. `v0.8.0`_ - 17-October-2021 diff --git a/environ/environ.py b/environ/environ.py index 42ebcd71..85e0ad3f 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -584,7 +584,15 @@ def cache_url_config(cls, url, backend=None): 'LOCATION': url.netloc + url.path, }) - if url.path and url.scheme in ['memcache', 'pymemcache']: + # urlparse('pymemcache://127.0.0.1:11211') + # => netloc='127.0.0.1:11211', path='' + # + # urlparse('pymemcache://127.0.0.1:11211') + # => netloc='memcached:11211', path='/' + # + # urlparse('memcache:///tmp/memcached.sock') + # => netloc='', path='/tmp/memcached.sock' + if not url.netloc and url.scheme in ['memcache', 'pymemcache']: config.update({ 'LOCATION': 'unix:' + url.path, }) diff --git a/tests/test_cache.py b/tests/test_cache.py index 44ef46c9..132266e6 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -68,6 +68,9 @@ def test_base_options_parsing(): ('pymemcache://127.0.0.1:11211', PYMEMCACHE_DRIVER, '127.0.0.1:11211'), + ('pymemcache://memcached:11211/?key_prefix=ci', + PYMEMCACHE_DRIVER, + 'memcached:11211'), ], ids=[ 'dbcache', @@ -84,6 +87,7 @@ def test_base_options_parsing(): 'memcached_multiple', 'memcached', 'pylibmccache', + 'pylibmccache_trailing_slash', ], ) def test_cache_parsing(url, backend, location): From 0a22b163ff69051e3e9bbf3a6f400b4e20b41c52 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 00:04:35 +0300 Subject: [PATCH 64/71] Update documentation --- environ/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 85e0ad3f..acdfb153 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -587,7 +587,7 @@ def cache_url_config(cls, url, backend=None): # urlparse('pymemcache://127.0.0.1:11211') # => netloc='127.0.0.1:11211', path='' # - # urlparse('pymemcache://127.0.0.1:11211') + # urlparse('pymemcache://memcached:11211/?key_prefix=ci') # => netloc='memcached:11211', path='/' # # urlparse('memcache:///tmp/memcached.sock') From bf45358a48444682b85ed40362a1f26e04d1ae89 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:15:37 +0300 Subject: [PATCH 65/71] Bump version --- CHANGELOG.rst | 10 ++++++++-- environ/__init__.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd34e763..093e23b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,8 +5,14 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.8.1`_ - 20-October-2021 +`v0.9.0`_ - 00-Unreleased-2021 +------------------------------ + +- WIP + +`v0.8.1`_ - 20-October-2021 +--------------------------- Fixed +++++ - Fixed "Invalid line" spam logs on blank lines in env file @@ -16,7 +22,7 @@ Fixed `v0.8.0`_ - 17-October-2021 ------------------------------- +--------------------------- Added +++++ - Log invalid lines when parse .env file diff --git a/environ/__init__.py b/environ/__init__.py index 97733f4d..e1d99cc4 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.8.1' +__version__ = '0.9.0' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' From 38c716614bd39dceaa637fa3d2593db78c65cc9d Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:17:43 +0300 Subject: [PATCH 66/71] Update documentation --- environ/fileaware_mapping.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/environ/fileaware_mapping.py b/environ/fileaware_mapping.py index e26ee8d1..706b92cd 100644 --- a/environ/fileaware_mapping.py +++ b/environ/fileaware_mapping.py @@ -1,3 +1,13 @@ +# This file is part of the django-environ. +# +# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2013-2021, Daniele Faraglia +# +# For the full copyright and license information, please view +# the LICENSE.txt file that was distributed with this source code. + +"""Docker-style file variable support module.""" + import os from collections.abc import MutableMapping From 6832d5538400fb51b20bc03b342dbf56a32fdcdd Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:29:10 +0300 Subject: [PATCH 67/71] Bump Python version used on CI --- .github/workflows/build.yml | 8 ++++---- .github/workflows/ci.yml | 2 +- .github/workflows/cs.yml | 4 ++-- .github/workflows/docs.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe711895..e594329f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,10 +29,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2.3.5 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.2 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | @@ -69,10 +69,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2.3.5 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.2 with: - python-version: '3.9' + python-version: '3.10' - name: Install in dev mode run: python -m pip install -e . diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c836edc9..7f30ed06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - '3.7' - '3.8' - '3.9' - - '3.10.0-beta - 3.10' + - '3.10' - 'pypy-3.7' os: [ ubuntu-latest, macos-latest, windows-latest ] diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index e4283d10..1d486450 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -23,10 +23,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2.3.5 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.2 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5c68593c..9db9696d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,10 +23,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2.3.5 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.2 with: - python-version: '3.8' + python-version: '3.10' - name: Install dependencies run: | From 2f7ada3d2f8942c681fbb4a3c3f7811b1022f2ac Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:29:30 +0300 Subject: [PATCH 68/71] Update RTD configuration --- .readthedocs.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 78a20224..91a97dae 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,13 +6,18 @@ # For the full copyright and license information, please view # the LICENSE.txt file that was distributed with this source code. +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + --- version: 2 -python: - # Keep version in sync with tox.ini (testenv:docs) and - # docs.yml (GitHub Action Workflow). - version: '3.8' +build: + os: ubuntu-20.04 + tools: + python: '3.10' + +python: install: - method: pip path: . From 67cc5411518ae99f020fdd9c1b93c1a80602eb64 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:48:46 +0300 Subject: [PATCH 69/71] Bump Python version in tox --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 4877c140..760c790b 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ commands = flake8 environ setup.py description = Check external links in the package documentation # Keep basepython in sync with .readthedocs.yml and docs.yml # (GitHub Action Workflow). -basepython = python3.8 +basepython = python3.10 extras = docs commands = {envpython} -m sphinx \ @@ -92,7 +92,7 @@ isolated_build = true description = Build package documentation (HTML) # Keep basepython in sync with .readthedocs.yml and docs.yml # (GitHub Action Workflow). -basepython = python3.8 +basepython = python3.10 extras = docs commands = {envpython} -m sphinx \ From 37a62866173ee62eec508e171d873a5ba83a7933 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:49:59 +0300 Subject: [PATCH 70/71] Fix change log --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 093e23b3..c1790c05 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -262,6 +262,7 @@ Added - Initial release. +.. _v0.9.0: https://github.com/joke2k/django-environ/compare/v0.8.1...develop .. _v0.8.1: https://github.com/joke2k/django-environ/compare/v0.8.0...v0.8.1 .. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 From a8682d76f455c843fd69f5bdbe12866c2326590b Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 20 Oct 2021 23:50:23 +0300 Subject: [PATCH 71/71] Prettify tests output --- .readthedocs.yml | 2 ++ tests/test_search.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 91a97dae..27f93688 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -15,6 +15,8 @@ version: 2 build: os: ubuntu-20.04 tools: + # Keep version in sync with tox.ini (testenv:docs) and + # docs.yml (GitHub Action Workflow). python: '3.10' python: diff --git a/tests/test_search.py b/tests/test_search.py index a81471e5..0992bf98 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -43,6 +43,12 @@ def test_solr_multicore_parsing(solr_url): 'elasticsearch5_backend.Elasticsearch5SearchEngine'), ('elasticsearch7://127.0.0.1:9200/index', 'elasticsearch7_backend.Elasticsearch7SearchEngine'), + ], + ids=[ + 'elasticsearch', + 'elasticsearch2', + 'elasticsearch5', + 'elasticsearch7', ] ) def test_elasticsearch_parsing(url, engine):