Create interface for manipulating objects in github files

* Create new interface for manipulating objects in files

* Update README with new interface

* Add comment explaining YAML representer

* Add test for BaseFile

* Make load/dump methods internal

README.md

@@ -34,26 +34,17 @@ class PreScale(Transformation):
         self.environments = args.environments
 
     def run(self):
-        changes = False
-        for f in self.repo.files:
-            if not re.match(f'overlays/{self.environments}/envconfig-values.yaml', f.path):
-                continue
-
-            k8s_patches = list(yaml.safe_load_all(f.decoded_content))
-            for r in k8s_patches:
-                if r['kind'] != 'HorizontalPodAutoscaler':
-                    continue
+        for env in self.environments:
+            file = self.repo.get_objects(f'overlays/{self.environments}/envconfig-values.yaml')
 
-                if r['spec']['minReplicas'] != r['spec']['maxReplicas']:
-                    r['spec']['maxReplicas'] = r['spec']['minReplicas']
-                    message = f"Setting maxRelicas = minReplicas = {r['spec']['minReplicas']}"
-                    changes = True
+            for obj in file:
+                if obj['kind'] != 'HorizontalPodAutoscaler':
+                    continue
 
-                break
+                if obj['spec']['minReplicas'] != obj['spec']['maxReplicas']:
+                    obj['spec']['maxReplicas'] = obj['spec']['minReplicas']
 
-            if changes:
-                file_str = yaml.dump_all(k8s_patches, default_flow_style=False, explicit_start=True)
-                self.repo.update_file(f, file_str, message, self.dry_run)
+            file.save(f'Setting maxRelicas = minReplicas = {obj['spec']['minReplicas']}', self.dry_run)
 
 
 if __name__ == '__main__':

gordian/files/__init__.py

@@ -0,0 +1,3 @@
+from .base_file import BaseFile
+from .yaml_file import YamlFile
+from .json_file import JsonFile

gordian/files/base_file.py

@@ -0,0 +1,32 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+class BaseFile:
+
+    def __init__(self, github_file, repo, start=0):
+        self.github_file = github_file
+        self.repo = repo
+        self.num = start
+        self.file_contents = github_file.decoded_content
+        self.objects = self._load_objects()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.num == len(self.objects):
+            raise StopIteration
+
+        num = self.num
+        self.num += 1
+        return self.objects[num]
+
+    def save(self, message, dry_run):
+        self.repo.update_file(self.github_file, self._dump(), message, dry_run)
+
+    def _load_objects(self):
+        raise NotImplementedError
+
+    def _dump(self):
+        raise NotImplementedError

gordian/files/json_file.py

@@ -0,0 +1,13 @@
+from . import BaseFile
+import json
+
+class JsonFile(BaseFile):
+
+    def __init__(self, github_file, repo):
+        super().__init__(github_file, repo)
+
+    def _load_objects(self):
+        return json.loads(self.file_contents)
+
+    def _dump(self):
+        return json.dumps(self.objects, indent=4)

gordian/files/yaml_file.py

@@ -0,0 +1,19 @@
+from . import BaseFile
+import yaml
+
+class YamlFile(BaseFile):
+
+    def __init__(self, github_file, repo):
+        super().__init__(github_file, repo)
+        yaml.add_representer(type(None), represent_none)
+
+    def _load_objects(self):
+        return list(yaml.safe_load_all(self.file_contents))
+
+    def _dump(self):
+        return yaml.dump_all(self.objects, default_flow_style=False, explicit_start=True)
+
+def represent_none(self, _):
+    # Disable dumping 'null' string for null values
+    # Taken from here: https://stackoverflow.com/a/41786451
+    return self.represent_scalar('tag:yaml.org,2002:null', '')

gordian/gordian.py

@@ -124,7 +124,6 @@ def apply_transformations(args, transformations):
     for repo_name in data:
         logger.info(f'Processing repo: {repo_name}')
         repo = Repo(repo_name, github_api_url=args.github_api, branch=args.branch)
-        repo.get_files()
         for transformation in transformations:
             transformation(args, repo).run()
         if repo.dirty:

gordian/repo.py

@@ -3,6 +3,7 @@
 import datetime
 import logging
 import os
+from gordian.files import YamlFile, JsonFile
 
 logger = logging.getLogger(__name__)
 
@@ -11,7 +12,7 @@
 
 class Repo:
 
-    def __init__(self, repo_name, github_api_url=None, branch=None, git=None):
+    def __init__(self, repo_name, github_api_url=None, branch=None, git=None, files=[]):
         if github_api_url is None:
             self.github_api_url = BASE_URL
         else:
@@ -24,8 +25,8 @@ def __init__(self, repo_name, github_api_url=None, branch=None, git=None):
                 git = Github(base_url=self.github_api_url, login_or_token=os.environ['GIT_USERNAME'], password=os.environ['GIT_PASSWORD'])
 
         self.repo_name = repo_name
-        self.repo = git.get_repo(repo_name)
-        self.files = []
+        self._repo = git.get_repo(repo_name)
+        self.files = files
         self.version_file = None
         self.branch_exists = False
         self.dirty = False
@@ -35,29 +36,45 @@ def __init__(self, repo_name, github_api_url=None, branch=None, git=None):
         else:
             self.branch_name = f"refs/heads/{datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S.%f')}"
 
-    def get_files(self):
-        if self.files:
-            return self.files
-
-        contents = self.repo.get_contents("")
-        while contents:
-            file_content = contents.pop(0)
-            if file_content.path == 'version':
-                self.version_file = file_content
-            if file_content.type == 'dir':
-                contents.extend(self.repo.get_contents(file_content.path))
-            else:
-                self.files.append(file_content)
+    def get_objects(self, filename, klass=None):
+        file = self.find_file(filename)
+
+        if file is None:
+            raise FileNotFoundError
+
+        if klass:
+            return klass(file, self)
 
-    def get_file(self, filename):
+        _, ext = os.path.splitext(filename)
+
+        if ext in ('.yaml', '.yml'):
+            return YamlFile(file, self)
+        if ext in ('.json'):
+            return JsonFile(file, self)
+
+    def get_files(self):
+        if not self.files:
+            contents = self._repo.get_contents("")
+            while contents:
+                file_content = contents.pop(0)
+                if file_content.path == 'version':
+                    self.version_file = file_content
+                if file_content.type == 'dir':
+                    contents.extend(self._repo.get_contents(file_content.path))
+                else:
+                    self.files.append(file_content)
+
+        return self.files
+
+    def find_file(self, filename):
         for file in self.get_files():
             if file.path == filename:
                 return file
 
     def make_branch(self):
-        sb = self.repo.get_branch('master')
+        sb = self._repo.get_branch('master')
         try:
-            ref = self.repo.create_git_ref(ref=self.branch_name, sha=sb.commit.sha)
+            ref = self._repo.create_git_ref(ref=self.branch_name, sha=sb.commit.sha)
         except GithubException as e:
             print(f"Branch {self.branch_name} already exists in github")
         self.branch_exists = True
@@ -98,7 +115,7 @@ def update_file(self, repo_file, content, message, dry_run=False):
         if not self.branch_exists:
             self.make_branch()
 
-        self.repo.update_file(
+        self._repo.update_file(
             repo_file.path,
             message,
             content,
@@ -116,7 +133,7 @@ def create_file(self, path, contents, message, dry_run=False):
         if not self.branch_exists:
             self.make_branch()
 
-        self.repo.create_file(
+        self._repo.create_file(
             path,
             message,
             contents,
@@ -133,7 +150,7 @@ def delete_file(self, file, message, dry_run=False):
         if not self.branch_exists:
             self.make_branch()
 
-        self.repo.delete_file(
+        self._repo.delete_file(
             file.path,
             message,
             file.sha,

setup.py

@@ -3,7 +3,7 @@
 setup_reqs = ["pytest-cov", "pytest-runner", "flake8"]
 setuptools.setup(
     name="gordian",
-    version="1.0.0",
+    version="1.1.0",
     author="Intuit",
     author_email="cg-sre@intuit.com",
     description="A tool to search and replace YAML files in a Git repo",

tests/test_base_file.py

@@ -0,0 +1,17 @@
+import unittest
+from unittest.mock import MagicMock
+from gordian.repo import Repo
+from gordian.files import YamlFile
+from .utils import Utils
+
+class TestBaseFile(unittest.TestCase):
+
+    def setUp(self):
+        self.github_file = Utils.create_github_content_file()
+        self.mock_git = MagicMock()
+        self.repo = Repo('test', git=self.mock_git)
+        self.base_file = YamlFile(self.github_file, self.repo)
+
+    def test_iterable(self):
+        assert(iter(self.base_file))
+

tests/test_repo.py

@@ -1,7 +1,9 @@
 import unittest
+import pytest
 from gordian.repo import Repo
 from unittest.mock import MagicMock, patch
-
+from gordian.files import YamlFile
+from .utils import Utils
 
 class TestRepo(unittest.TestCase):
 
@@ -11,6 +13,8 @@ def setUp(self, mock_git):
         self.mock_repo = MagicMock()
         self.mock_branches = MagicMock()
         self.repo = Repo('test', git=self.mock_git)
+        self.repo.files.append(Utils.create_github_content_file())
+
         self.mock_repo.get_branches.return_value = self.mock_branches
         mock_git.get_repo.return_value = self.mock_repo
         self.instance = Repo(None, branch='', git=mock_git)
@@ -29,5 +33,13 @@ def test_default_github_url(self):
         self.assertEqual(self.repo.github_api_url, 'https://api.github.com')
 
     def test_override_github_url(self):
-        self.repo = Repo('test', github_api_url='https://test.github.com', git=self.mock_git)
-        self.assertEqual(self.repo.github_api_url, 'https://test.github.com')
+        repo = Repo('test', github_api_url='https://test.github.com', git=self.mock_git)
+        self.assertEqual(repo.github_api_url, 'https://test.github.com')
+
+    def test_get_object_does_not_exist(self):
+        with pytest.raises(FileNotFoundError):
+            self.repo.get_objects('test')
+
+    def test_get_existing_object(self):
+        contents = self.repo.get_objects('/content.yaml')
+        assert(isinstance(contents, YamlFile))

tests/test_transformations.py

@@ -18,7 +18,7 @@ def setUp(self, mock_git):
         self.mock_branches = MagicMock()
         self.mock_repo.get_branches.return_value = self.mock_branches
         mock_git.get_repo.return_value = self.mock_repo
-        self.instance = Repo(None, branch='', git=mock_git)
+        self.instance = Repo(None, branch='', git=mock_git, files=[])
         self.instance.branch_exists = False
         f = open('./tests/fixtures/content.yaml', 'r')
         contents = f.read()

tests/utils.py

@@ -0,0 +1,10 @@
+import base64
+from github import ContentFile
+
+class Utils:
+
+    def create_github_content_file():
+        f = open('./tests/fixtures/content.yaml', 'r')
+        contents = str(base64.b64encode(bytearray(f.read(), 'utf-8')), 'utf-8')
+        attributes = {'name':'content.yaml', 'path':'/content.yaml','encoding':'base64','content': contents}
+        return ContentFile.ContentFile(None, {}, attributes, completed=True)