feat: add support for files upload and apikey auth (#39)

* build: update build version-s
* feat: add apikey authentication
* refactor: use json dumps default to mitigate TypeError
* refactor: use redacting log formatter
* refactor: fix logging
* fix: remove class items to redact
* feat: add support for files
* build: use pyb test coverage
--------
Signed-off-by: Emilio Reyes <soda480@gmail.com>

.github/workflows/main.yml

@@ -20,37 +20,3 @@ jobs:
       - name: build rest3client ${{ matrix.version }} image
         run:
           docker image build --target build-image --build-arg PYTHON_VERSION=${{ matrix.version }} -t rest3client:${{ matrix.version }} .
-      - name: save rest3client ${{ matrix.version }} image
-        if: ${{ matrix.version == '3.9' }}
-        run: |
-          mkdir -p images
-          docker save --output images/rest3client-${{ matrix.version }}.tar rest3client:${{ matrix.version }}
-      - name: upload rest3client ${{ matrix.version }} image artifact
-        if: ${{ matrix.version == '3.9' }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: image
-          path: images/rest3client-${{ matrix.version }}.tar
-  coverage:
-    name: Publish Code Coverage Report
-    needs: build-images
-    runs-on: ubuntu-20.04
-    steps:
-      - name: download image artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: image
-          path: images/
-      - name: load image
-        run:
-          docker load --input images/rest3client-3.9.tar
-      - name: prepare report
-        run: |
-          ID=$(docker create rest3client:3.9)
-          docker cp $ID:/code/target/reports/rest3client_coverage.xml rest3client_coverage.xml
-          sed -i -e 's,filename="rest3client/,filename="src/main/python/rest3client/,g' rest3client_coverage.xml
-      - name: upload report
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: rest3client_coverage.xml

README.md

@@ -1,6 +1,7 @@
 # rest3client
 [![GitHub Workflow Status](https://github.com/soda480/rest3client/workflows/build/badge.svg)](https://github.com/soda480/rest3client/actions)
-[![Code Coverage](https://codecov.io/gh/soda480/rest3client/branch/master/graph/badge.svg)](https://codecov.io/gh/soda480/rest3client)
+[![coverage](https://img.shields.io/badge/coverage-99%25-brightgreen)](https://pybuilder.io/)
+[![complexity](https://img.shields.io/badge/complexity-A-brightgreen)](https://radon.readthedocs.io/en/latest/api.html#module-radon.complexity)
 [![vulnerabilities](https://img.shields.io/badge/vulnerabilities-None-brightgreen)](https://pypi.org/project/bandit/)
 [![PyPI version](https://badge.fury.io/py/rest3client.svg)](https://badge.fury.io/py/rest3client)
 [![python](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-teal)](https://www.python.org/downloads/)
@@ -56,6 +57,11 @@ The examples below show how RESTclient can be used to consume the GitHub REST AP
 
 # jwt authentication
 >>> client = RESTclient('my-api.my-company.com', jwt='--my-jwt--')
+
+# api key authentication
+>>> client = RESTclient('my-api.my-company.com', api_key='--my-api-key--')
+# or some systems use apikey header
+>>> client = RESTclient('my-api.my-company.com', apikey='--my-api-key--')
 ```
 
 `GET` request

build.py

@@ -30,7 +30,7 @@
 authors = [Author('Emilio Reyes', 'emilio.reyes@intel.com')]
 summary = 'An abstraction of the requests library providing a simpler API for consuming HTTP REST APIs'
 url = 'https://github.com/soda480/rest3client'
-version = '0.6.0'
+version = '0.7.0'
 default_task = [
     'clean',
     'analyze',
@@ -79,4 +79,4 @@ def set_properties(project):
     project.set_property('radon_break_build_average_complexity_threshold', 3.6)
     project.set_property('radon_break_build_complexity_threshold', 14)
     project.set_property('bandit_break_build', True)
-    project.set_property('anybadge_exclude', 'coverage, complexity')
+    project.set_property('anybadge_complexity_use_average', True)

src/main/python/rest3client/restclient.py

@@ -31,21 +31,29 @@
 logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.CRITICAL)
 
 
+class RedactingFormatter(logging.Formatter):
+
+    def __init__(self, orig_formatter, secrets=None):
+        self.orig_formatter = orig_formatter
+        self._secrets = secrets
+
+    def format(self, record):
+        msg = self.orig_formatter.format(record)
+        if self._secrets:
+            for secret in self._secrets:
+                if secret in msg:
+                    msg = msg.replace(secret, "[REDACTED]")
+        return msg
+
+    def __getattr__(self, attr):
+        return getattr(self.orig_formatter, attr)
+
+
 class RESTclient():
     """ class exposing abstracted requests-based http verb apis
     """
 
     cabundle = '/etc/ssl/certs/ca-certificates.crt'
-    items_to_redact = [
-        'Authorization',
-        'Auth',
-        'x-api-key',
-        'Password',
-        'password',
-        'JWT',
-        'Token',
-        'token'
-    ]
 
     def __init__(self, hostname, **kwargs):
         """ class constructor
@@ -60,6 +68,7 @@ def __init__(self, hostname, **kwargs):
         self.password = kwargs.get('password')
 
         self.api_key = kwargs.get('api_key')
+        self.apikey = kwargs.get('apikey')
 
         self.bearer_token = kwargs.get('bearer_token')
 
@@ -77,21 +86,41 @@ def __init__(self, hostname, **kwargs):
         self.retries = kwargs.get('retries', [])
         self.decorate_retries()
 
+        items_to_redact = [
+            self.password,
+            self.api_key,
+            self.apikey,
+            self.bearer_token,
+            self.token,
+            self.jwt,
+            self.certpass
+        ]
+        if self.username and self.password:
+            self.basic = base64.b64encode((f'{self.username}:{self.password}').encode())
+            self.basic = f'{self.basic}'.replace('b\'', '').replace('\'', '')
+            items_to_redact.append(self.basic)
+        items_to_be_redacted = [item for item in items_to_redact if item]
+        for handler in logger.root.handlers:
+            handler.setFormatter(RedactingFormatter(handler.formatter, secrets=items_to_be_redacted))
+
     def get_headers(self, **kwargs):
         """ return headers to pass to requests method
         """
         headers = kwargs.get('headers', {})
 
-        if 'Content-Type' not in headers:
+        if 'files' not in kwargs and 'Content-Type' not in headers:
+            # do not set Content-Type when files are being posted
             headers['Content-Type'] = 'application/json'
 
         if self.username and self.password:
-            basic = base64.b64encode((f'{self.username}:{self.password}').encode())
-            headers['Authorization'] = f'Basic {basic}'.replace('b\'', '').replace('\'', '')
+            headers['Authorization'] = f'Basic {self.basic}'
 
         if self.api_key:
             headers['x-api-key'] = self.api_key
 
+        if self.apikey:
+            headers['apikey'] = self.apikey
+
         if self.bearer_token:
             headers['Authorization'] = f'Bearer {self.bearer_token}'
 
@@ -104,37 +133,30 @@ def get_headers(self, **kwargs):
         return headers
 
     def get_arguments(self, endpoint, kwargs):
-        """ return key word arguments to pass to requests method
+        """ update kwargs with required values to pass to requests method
         """
-        arguments = copy.deepcopy(kwargs)
-
         headers = self.get_headers(**kwargs)
-        if 'headers' not in arguments:
-            arguments['headers'] = headers
+        if 'headers' not in kwargs:
+            kwargs['headers'] = headers
         else:
-            arguments['headers'].update(headers)
+            kwargs['headers'].update(headers)
 
-        if 'verify' not in arguments or arguments.get('verify') is None:
-            arguments['verify'] = self.cabundle
+        if 'verify' not in kwargs or kwargs.get('verify') is None:
+            kwargs['verify'] = self.cabundle
 
         if endpoint.startswith('http'):
-            arguments['address'] = endpoint
+            kwargs['address'] = endpoint
         else:
-            arguments['address'] = f'https://{self.hostname}{endpoint}'
-        arguments.pop('raw_response', None)
-        return arguments
+            kwargs['address'] = f'https://{self.hostname}{endpoint}'
 
     def log_request(self, function_name, arguments, noop):
         """ log request function name and redacted arguments
         """
-        redacted_arguments = self.redact(arguments)
-        try:
-            redacted_arguments = json.dumps(redacted_arguments, indent=2, sort_keys=True)
-        except TypeError:
-            pass
-
+        redacted_arguments = json.dumps(arguments, indent=2, sort_keys=True, default=str)
         cert = f'\nCERT: {self.certfile}' if self.certfile else ''
-        logger.debug(f"\n{function_name}: {arguments['address']} NOOP: {noop}\n{redacted_arguments}{cert}")
+        if function_name.startswith('_'):
+            function_name = function_name[1:]
+        logger.debug(f"\n{function_name}: {arguments['address']}   NOOP: {noop}\n{redacted_arguments}{cert}")
 
     def get_error_message(self, response):
         """ return error message from response
@@ -155,7 +177,7 @@ def get_response(self, response, **kwargs):
         logger.debug('processing response')
 
         if not response.ok:
-            logger.debug('response was not OK')
+            logger.debug('response was NOT OK')
             error_message = self.get_error_message(response)
             logger.debug(f'{error_message}: {response.status_code}')
             response.raise_for_status()
@@ -184,11 +206,13 @@ def _request_handler(self, endpoint, **kwargs):
             """ decorator method to prepare and handle requests and responses
             """
             noop = kwargs.pop('noop', False)
-            arguments = self.get_arguments(endpoint, kwargs)
-            self.log_request(function.__name__.upper(), arguments, noop)
+            raw_response = kwargs.pop('raw_response', None)
+            self.get_arguments(endpoint, kwargs)
+            self.log_request(function.__name__.upper(), kwargs, noop)
             if noop:
                 return
-            response = function(self, endpoint, **arguments)
+            response = function(self, endpoint, **kwargs)
+            kwargs['raw_response'] = raw_response
             return self.get_response(response, **kwargs)
         return _request_handler
 
@@ -376,30 +400,6 @@ def decorate_retries(self):
             self.delete = retry(**retry_kwargs)(self.delete)
             self.patch = retry(**retry_kwargs)(self.patch)
 
-    @classmethod
-    def redact(cls, items):
-        """ return redacted copy of items dictionary
-        """
-        def _redact(items):
-            """ redact private method
-            """
-            if isinstance(items, dict):
-                for item_to_redact in cls.items_to_redact:
-                    if item_to_redact in items:
-                        items[item_to_redact] = '[REDACTED]'
-                for item in items.values():
-                    _redact(item)
-            elif isinstance(items, Iterable) and not isinstance(items, str):
-                for item in items:
-                    _redact(item)
-
-        scrubbed = copy.deepcopy(items)
-        if 'address' in scrubbed:
-            del scrubbed['address']
-        for value in scrubbed.values():
-            _redact(value)
-        return scrubbed
-
     @staticmethod
     def get_loggable_kwargs(kwargs):
         """ return copy of kwargs that can be logged (serializable)

src/unittest/python/test_RESTcli.py

@@ -18,12 +18,9 @@
 from mock import mock_open
 from mock import call
 from mock import Mock
-
 from rest3client import RESTcli
 from argparse import Namespace
 import sys
-import logging
-logger = logging.getLogger(__name__)
 
 
 class TestRESTcli(unittest.TestCase):