diff --git a/.gitignore b/.gitignore index 28982b7..bd7a118 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ __pycache__/ node_modules .idea *.lock -*.pyc \ No newline at end of file +*.pyc diff --git a/Makefile b/Makefile index bce5edb..350f780 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,17 @@ SHELL := /bin/bash .PHONY: backend backend: pip install -r backend/requirements.txt - python3 backend/web_server.py + python3 backend/server.py .PHONY: frontend frontend: yarn --cwd frontend install yarn --cwd frontend run serve +.PHONY: webhook +webhook: + smee --url https://smee.io/UUkxjK9NwrD3pnTH --path /webhook_handler --port 5000 + .PHONY: db db: mongod --config /usr/local/etc/mongod.conf diff --git a/README.md b/README.md index 710afe4..f1695ee 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,13 @@ To clean the database document collection, run make db_clean ``` +### Setting the webhook redirect + +To set the webhook redirect to localhost: +```bash +make webhook +``` + #### Check out the app Visit http://localhost:8080/ @@ -70,8 +77,26 @@ pip freeze > backend/requirements.txt pip install -r backend/requirements.txt ``` +### Clone the test folder locally + +```bash +git clone git@github.com:codersdoc/Test.git +``` + +### Change the setup of the Github app + +Go to this [link](https://github.com/settings/apps/tech-documentation). + +### Useful links + +* [Github enpoints available](https://developer.github.com/v3/apps/available-endpoints/). + + ## Miscallaneous ### Tech and library used * pyGithub +* mongo +* flask +* Vuejs diff --git a/backend/github_interface/git/git_diff_types/git_diff_hunk.py b/backend/github_interface/git/git_diff_types/git_diff_hunk.py index 569a32f..dc69bd4 100644 --- a/backend/github_interface/git/git_diff_types/git_diff_hunk.py +++ b/backend/github_interface/git/git_diff_types/git_diff_hunk.py @@ -45,6 +45,7 @@ def count_line_change_after_exclusive(self, line_number): def __count_line_change_helper(self, line_number, is_count_before): current_old_line_number = self.get_old_start_line() - 1 total_line_change = 0 + counter_reseted = False for code_line in self.code_lines: if code_line.state == GitDiffCodeLineState.UNCHANGED: @@ -55,10 +56,11 @@ def __count_line_change_helper(self, line_number, is_count_before): elif code_line.state == GitDiffCodeLineState.ADDED: total_line_change += 1 - if current_old_line_number == line_number: + if current_old_line_number == line_number and not counter_reseted: if is_count_before: break else: + counter_reseted = True total_line_change = 0 return total_line_change diff --git a/backend/github_interface/github_types/github_commit_file.py b/backend/github_interface/github_types/github_commit_file.py index 2a8cdee..5008558 100644 --- a/backend/github_interface/github_types/github_commit_file.py +++ b/backend/github_interface/github_types/github_commit_file.py @@ -3,21 +3,29 @@ class GithubCommitFile(GithubFile): - def __init__(self, content_file, previous_path, patch): + def __init__(self, content_file, path, previous_path, patch, is_deleted): GithubFile.__init__(self, content_file) - self._previous_path = previous_path + self.__path = path + self.__has_path_changed = not (previous_path is None) + self.__previous_path = self.__path if previous_path is None else previous_path self.__git_diff_parser = GitDiffParser(patch) + self.__is_deleted = is_deleted @property def previous_path(self): - return self._previous_path + return self.__previous_path + + @property + def has_path_changed(self): + return self.__has_path_changed + + @property + def is_deleted(self): + return self.__is_deleted def calculate_updated_line_range(self, start_line, end_line): return self.__git_diff_parser.calculate_updated_line_range(start_line, end_line) - def has_path_changed(self): - return self.previous_path is not None - def to_json(self): new_json = { "previous_path": self.previous_path diff --git a/backend/github_interface/github_types/github_repository.py b/backend/github_interface/github_types/github_repository.py index 367e425..3ac64ca 100644 --- a/backend/github_interface/github_types/github_repository.py +++ b/backend/github_interface/github_types/github_repository.py @@ -45,9 +45,11 @@ def get_commit_files(self, branch_name="master", sha=None): for file in commit.files: if file.status == "removed": file_object = None + is_deleted = True else: file_object = self.__get_file_object(file.filename) - files.append(GithubCommitFile(file_object, file.previous_filename, file.patch)) + is_deleted = False + files.append(GithubCommitFile(file_object, file.filename, file.previous_filename, file.patch, is_deleted)) return files diff --git a/backend/github_interface/hooks/hook_manager.py b/backend/github_interface/hooks/hook_manager.py deleted file mode 100644 index 0abba1a..0000000 --- a/backend/github_interface/hooks/hook_manager.py +++ /dev/null @@ -1,32 +0,0 @@ -import json - -import requests - -""" -https://developer.github.com/v3/repos/hooks/ -""" - - -def get_hooks(oauth_token, owner, repo): - return requests.get( - "https://api.github.com/repos/{}/{}/hooks".format(owner, repo), - headers={'Authorization': 'token {}'.format(oauth_token)} - ).json() - - -def subscribe_hooks(oauth_token, owner, repo, url_postback): - return requests.post( - "https://api.github.com/repos/{}/{}/hooks".format(owner, repo), - data=json.dumps({ - "name": "web", - "active": True, - "events": [ - "pull_request" - ], - "config": { - "url": url_postback, - "content_type": "json" - } - }), - headers={'Authorization': 'token {}'.format(oauth_token)} - ).json() \ No newline at end of file diff --git a/backend/github_interface/interface.py b/backend/github_interface/interface.py index e75c720..2547531 100644 --- a/backend/github_interface/interface.py +++ b/backend/github_interface/interface.py @@ -1,25 +1,84 @@ -from github import Github +import hmac + +import requests +from github import Github, GithubIntegration +from hashlib import sha1 from github_interface.github_types.github_repository import GithubRepository +from mongo.credentials import CredentialsManager +from utils.constants import GITHUB_APP_IDENTIFIER class GithubInterface: - def __init__(self, access_token): - self.__github_account = Github(access_token) - self.__repo_cache = {} + __repo_cache = {} + + # TODO: use repo id and not repo name for the cache + @staticmethod + def get_repo(repo_name, is_user_access_token=False): + github_account = Github(GithubInterface.__fetch_access_token_from_db(is_user_access_token)) - def get_repo(self, repo_name): - if repo_name in self.__repo_cache: - repo = self.__repo_cache[repo_name] + if repo_name in GithubInterface.__repo_cache: + repo = GithubInterface.__repo_cache[repo_name] else: - self.__repo_cache[repo_name] = GithubRepository(self.__github_account.get_repo(repo_name)) - repo = self.__repo_cache[repo_name] + GithubInterface.__repo_cache[repo_name] = GithubRepository(github_account.get_repo(repo_name)) + repo = GithubInterface.__repo_cache[repo_name] return repo - def get_repos(self): + @staticmethod + def get_repos(is_user_access_token=False): + github_account = Github(GithubInterface.__fetch_access_token_from_db(is_user_access_token)) + repos = [] - for repo in self.__github_account.get_user().get_repos(): + + if is_user_access_token: + raw_repos = github_account.get_user().get_repos() + else: + raw_repos = github_account.get_installation(-1).get_repos() + + for repo in raw_repos: repos.append(GithubRepository(repo)) return repos + @staticmethod + def get_user_access_token(client_id, client_secret, code, redirect_uri): + params = { + "client_id": client_id, + "client_secret": client_secret, + "code": code, + "redirect_uri": redirect_uri + } + r = requests.get(url="https://github.com/login/oauth/access_token", params=params) + content = r.content.decode("utf-8") + user_access_token = content[content.find("access_token=") + 13: content.find("&")] + + return user_access_token + + @staticmethod + def get_user_installations(): + access_token = GithubInterface.__fetch_access_token_from_db(True) + response = requests.get(url="https://api.github.com/user/installations", + headers={ + "Authorization": "token " + access_token, + "Accept": "application/vnd.github.machine-man-preview+json" + }) + return response.json() + + @staticmethod + def get_installation_access_token(installation_id, private_key): + integration = GithubIntegration(str(GITHUB_APP_IDENTIFIER), private_key) + return integration.get_access_token(installation_id).token + + @staticmethod + def verify_signature(signature, body, github_webhook_secret): + computed_signature = "sha1=" + hmac.new(str.encode(github_webhook_secret), body, sha1).hexdigest() + return computed_signature == signature + + @staticmethod + def __fetch_access_token_from_db(is_user_access_token): + if is_user_access_token: + return CredentialsManager.read_credentials()["user_access_token"] + else: + return CredentialsManager.read_credentials()["installation_access_token"] + + diff --git a/backend/mongo/credentials.py b/backend/mongo/credentials.py new file mode 100644 index 0000000..35062b0 --- /dev/null +++ b/backend/mongo/credentials.py @@ -0,0 +1,29 @@ +import json + + +class CredentialsManager: + + @staticmethod + def write_credentials(user_access_token="", installation_access_token=""): + credentials = CredentialsManager.read_credentials() + + user_access_token = credentials[ + "user_access_token"] if "user_access_token" in credentials and not user_access_token else user_access_token + installation_access_token = credentials[ + "installation_access_token"] if "installation_access_token" in credentials and not installation_access_token else installation_access_token + + with open("backend/ressources/credentials.txt", "w") as file: + credentials = "{\"installation_access_token\":\"" + str( + installation_access_token) + "\",\"user_access_token\":\"" + str(user_access_token) + "\"}" + file.write(credentials) + file.close() + + @staticmethod + def read_credentials(): + with open("backend/ressources/credentials.txt", "r") as file: + file_content = file.read() + if file_content == "": + file_content = "{}" + file.close() + + return json.loads(file_content) \ No newline at end of file diff --git a/backend/mongo/models.py b/backend/mongo/models.py index 029438d..ccf0ac1 100644 --- a/backend/mongo/models.py +++ b/backend/mongo/models.py @@ -13,12 +13,13 @@ class FileReference: A FileReference is part of a Document, and references lines of code in repositories """ - def __init__(self, ref_id, repo, path, start_line, end_line): + def __init__(self, ref_id, repo, path, start_line, end_line, is_deleted): self.ref_id = ref_id self.repo = repo self.path = path self.start_line = start_line self.end_line = end_line + self.is_deleted = is_deleted def to_json(self): return { @@ -27,6 +28,7 @@ def to_json(self): 'path': self.path, 'start_line': self.start_line, 'end_line': self.end_line, + 'is_deleted': self.is_deleted } @staticmethod @@ -36,7 +38,8 @@ def from_json(file_ref): file_ref['repo'], file_ref['path'], int(file_ref['start_line']), - int(file_ref['end_line']) + int(file_ref['end_line']), + file_ref['is_deleted'] ) def __init__(self, name, content, references): @@ -47,6 +50,45 @@ def __init__(self, name, content, references): def insert(self): return self.COLLECTION.insert_one(self.to_json()) + @staticmethod + def __update(query, new_values): + return Document.COLLECTION.update_one(query, new_values) + + @staticmethod + def update_lines_ref(ref_id, new_start_line, new_end_line): + query = { "refs.ref_id": ref_id } + new_values = { + "$set": { + "refs.$.start_line": new_start_line, + "refs.$.end_line": new_end_line + } + } + + return Document.__update(query, new_values) + + @staticmethod + def update_path_ref(ref_id, path): + query = {"refs.ref_id": ref_id} + new_values = { + "$set": { + "refs.$.path": path + } + } + + return Document.__update(query, new_values) + + staticmethod + + def update_is_deleted_ref(ref_id, is_deleted): + query = {"refs.ref_id": ref_id} + new_values = { + "$set": { + "refs.$.is_deleted": is_deleted + } + } + + return Document.__update(query, new_values) + @staticmethod def find(name): doc = Document.COLLECTION.find_one({ diff --git a/backend/requirements.txt b/backend/requirements.txt index 1b66ac9..52b73b9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,17 +1,28 @@ +asn1crypto==0.24.0 certifi==2019.3.9 +cffi==1.12.3 chardet==3.0.4 Click==7.0 +cryptography==2.7 Deprecated==1.2.5 Flask==1.0.3 +GitHub-Flask==3.2.0 +github3.py==1.3.0 idna==2.8 itsdangerous==1.1.0 Jinja2==2.10.1 +jwcrypto==0.6.0 +jwt==0.6.1 MarkupSafe==1.1.1 +pycparser==2.19 PyGithub==1.43.7 Pygments==2.4.2 PyJWT==1.7.1 pymongo==3.8.0 +python-dateutil==2.8.0 requests==2.22.0 +six==1.12.0 +uritemplate==3.0.0 urllib3==1.25.3 Werkzeug==0.15.4 wrapt==1.11.1 diff --git a/backend/ressources/credentials.txt b/backend/ressources/credentials.txt new file mode 100644 index 0000000..dbc524b --- /dev/null +++ b/backend/ressources/credentials.txt @@ -0,0 +1 @@ +{"installation_access_token":"v1.065b6ed40c59378b45474b6b14493565275baffb","user_access_token":"d66a5217cc5b6fd2a18a61e3a7006b9e94a361c2"} \ No newline at end of file diff --git a/backend/ressources/github-private-key.pem b/backend/ressources/github-private-key.pem new file mode 100644 index 0000000..5842b9c --- /dev/null +++ b/backend/ressources/github-private-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA8UXZx+15S5llpuUKkFkLxXgTCGUSVVDm02WwASxi1JJzD/oL +aSXjyl2rujTCA8oTRB0zyeohf0LirQ+PdHjHR4YJPHVnast0pMoEfh0MEvLUGZEh +OFSdpd7QDWrr21V8SX/lUCznvyU71MpNTC/IwhjZh3/B0U3SrU6aztceHAEUK6r5 +bIJ31nJuOGPjqqYcfJjOR8DUAhqf/LAaSXaDQqaTHXFPLUKrNKIvjMP04yUywcf0 +G6DP8GarF0Qj87woZn3d18KrX7Vp5JzV7LsQz47xtutYky9K+dl2gU1ZZeOmDvVR +vtBfEXP0LNRCgISxWGZUwSm7E0SBibKY4oEmiwIDAQABAoIBAAFPg9NWMuZ6Otch +P2FxWmMEN/Y/tk3IVrinQMGA4DiPYxifHxi/H/GleJ1WVAd5PYmNLw7VusDaOCkA +gKL9VPfKfppZeOpXmJacklGtDre7ofNRmoCX1RNllnO8NPPIDxjHHRPGaqvbI+wP +/UOArvJ++A+IXiEo7xAJ8UVWXc+BH0u13WErKoj+juThGocuLTv94y9pd7x94eQi +Uai5IFyY+mgf9F60SsTFeKCEHj3d9HeilR/z3OD8uXspYWQzO62nnbIu4lGHlgF8 +DiOyAkQks3XfB4A/2pqPInECLavz4DdgSYcYzi98/Rb+IeuEjplg5Yv+iF+rjGPl +OYnCJqECgYEA+MQ9sQuEwloUAvNLURereJrvdkllnM6IBFWWwHMy5NAfvYtyd0zX +hnL3uUfqlrTl56qeF/rui5VWdgJoCEE/VvKsBLP6cci+08brKaRnUMgFoUrzH4VE +jHiUGNDOwYgTI8z1d7oolJTgEf97aDk+DX+Tgces7VYmxaIsrrnMmT0CgYEA+EnU +DKtimJs5GyKUryJK4ZpiBoUT6wGCgzED7TjvbE8L3WgHHd/xRoI2ZnU2Tl8BNc/v +yQvZ8FQoUe1op4EjvK3a8bvQflNvur1KZI1BFMYfLA4iYde6q4LluaGqsDi/IP96 +4xcczSbhIXixXWgC9d7wXGa70ehRd0VIerXfa2cCgYBhZpSxCU2FuzcyoIfQzG+6 +3Q79RWefqc3fxJMt7uzyYfrLgBnlVBTe84zC4sGbGGEb/9W+leVoiaQ8uFx7PvDJ +3mIzxTQ98Nemm6/fshsxqd9qc6oVoVxhk6SIwtjxNZIo5ksGAcF5y4CgC2QKPr9p +EZZzrfarRpwPrZvJHb5aEQKBgHjVvWx5EGAC0zUAjGn7f4PyVZiktX/e2Tyt4yJV +XjhQ9A5J7YS9kzfkcUNF8isME5Oz4hfvO5655nGQ4Cj9MX5HAlI5PIvuYWb5brYn +BLBuh4cyTcteaUvFRbYlFuPyihouHAlfGzZAoLpgebliwGYWnNXrbacHsHYicta9 +osErAoGAVZw0S5YLkAEoCo4etgAUuH81bOKVDMw7PxaTAVvu20MUXiCmycGNGYDM +WWQOj3xiA84eYOJN5KBpoHNYUQH8YDk1FumiQT8OQTb3PggSvQyU8kkWaeGCIE7q +rtuGI4ymeIIvO7pu4qhwKLPY5V7CtlDOa5nW97mWtS6hCaDLGRk= +-----END RSA PRIVATE KEY----- diff --git a/backend/server.py b/backend/server.py new file mode 100644 index 0000000..269abfd --- /dev/null +++ b/backend/server.py @@ -0,0 +1,14 @@ +from flask import Flask + +from server.web_server import web_server +from server.webhook_server import webhook_server +from utils.json.custom_json_encoder import CustomJsonEncoder + +app = Flask(__name__) +app.json_encoder = CustomJsonEncoder + +app.register_blueprint(webhook_server) +app.register_blueprint(web_server) + +if __name__ == '__main__': + app.run() diff --git a/backend/github_interface/hooks/__init__.py b/backend/server/__init__.py similarity index 100% rename from backend/github_interface/hooks/__init__.py rename to backend/server/__init__.py diff --git a/backend/server/web_server.py b/backend/server/web_server.py new file mode 100644 index 0000000..fe2dde5 --- /dev/null +++ b/backend/server/web_server.py @@ -0,0 +1,213 @@ +import copy +import uuid + +from flask import jsonify, request, abort, Response, Blueprint +from pygments import highlight +from pygments.formatters.html import HtmlFormatter +from pygments.lexers import get_lexer_for_filename +from pygments.lexers.special import TextLexer +from pygments.util import ClassNotFound + +from github_interface.interface import GithubInterface +from mongo.credentials import CredentialsManager +from mongo.models import Document +from utils import code_formatter +from utils.constants import SECRET_PASSWORD_FORGERY, CLIENT_ID, CLIENT_SECRET, REDIRECT_URL_LOGIN +from utils.file_interface import FileInterface + +web_server = Blueprint('web_server', __name__,) + +@web_server.route("/repos") +def repos(): + # Get the repository list + repo_names = [r.full_name for r in GithubInterface.get_repos()] + + # Return the response + return __create_response(repo_names) + +@web_server.route("/file") +def file(): + # Get the repository + repo_name = request.args.get('repo') + + if not repo_name: + return abort(400, "A repo should be specified") + + repo = GithubInterface.get_repo(repo_name) + + # Get the content at path + path_arg = request.args.get('path') + path = path_arg if path_arg else "" + + # TODO: fix this so we don't have to deepcopy + repo_object = copy.deepcopy(repo.get_content_at_path(path)) + + # Syntax highlighting for file + if repo_object.type == 'file': + try: + lexer = get_lexer_for_filename(path) + except ClassNotFound: + lexer = TextLexer() # use a generic lexer if we can't find anything + + formatter = HtmlFormatter(noclasses=True, linenos='table', linespans='code-line') + repo_object.content = highlight(repo_object.content, lexer, formatter) + + # Return the response + return __create_response(repo_object) + +@web_server.route("/save", methods=['POST', 'OPTIONS']) +def save(): + if request.method == 'OPTIONS': + return __create_option_response() + + if Document.find(request.get_json().get('name')): + return abort(400, 'Document name already exists') + + doc = Document.from_json(request.get_json()) + doc.insert() + + return __create_response({}) + + +@web_server.route("/docs") +def docs(): + docs = Document.get_all() + + return __create_response([doc.to_json() for doc in docs]) + +@web_server.route("/render") +def render(): + name = request.args.get('name') + + # Get the documentation doc + doc = Document.find(name) + + references = {} + + for ref in doc.references: + repo = GithubInterface.get_repo(ref.repo) + + content = '\n'.join(repo.get_lines_at_path(ref.path, ref.start_line, ref.end_line)) + formatted_code = code_formatter.format(ref.path, content, ref.start_line) + + references[ref.ref_id] = { + 'code': formatted_code, + 'repo': ref.repo, + 'path': ref.path, + 'startLine': ref.start_line, + 'endLine': ref.end_line, + } + + return __create_response({ + 'name': name, + 'content': doc.content, + 'refs': references + }) + + + +# TODO: similar to render -> refactor later +@web_server.route("/lines") +def get_lines(): + repo = request.args.get('repo') + path = request.args.get('path') + start_line = int(request.args.get('startLine')) + end_line = int(request.args.get('endLine')) + + repository = GithubInterface.get_repo(repo) + content = ''.join(repository.get_content_at_path(path).content.splitlines(keepends=True)[start_line - 1: end_line]) + + try: + lexer = get_lexer_for_filename(path) + except ClassNotFound: + lexer = TextLexer() # use a generic lexer if we can't find anything + + formatter = HtmlFormatter(noclasses=True, linenos='table', linespans='code-line', linenostart=start_line) + code = highlight(content, lexer, formatter) + + return __create_response({ + 'ref_id': str(uuid.uuid1()), # generate a unique id for the reference + 'code': code, + 'repo': repo, + 'path': path, + 'startLine': start_line, + 'endLine': end_line, + }) + +@web_server.route("/auth/github/callback", methods=['POST', 'OPTIONS']) +def auth_github_callback(): + if request.method == 'OPTIONS': + return __create_option_response() + + temporary_code = request.args.get('code') + state = request.args.get('state') + + if state != SECRET_PASSWORD_FORGERY: + abort(401) + + user_access_token = GithubInterface.get_user_access_token(CLIENT_ID, CLIENT_SECRET, temporary_code, REDIRECT_URL_LOGIN) + CredentialsManager.write_credentials(user_access_token=user_access_token) + + return __create_response({}) + +@web_server.route("/installs") +def installs(): + user_access_token = CredentialsManager.read_credentials()["user_access_token"] + if not user_access_token: + user_installations = {"installations": []} + else: + user_installations = GithubInterface.get_user_installations() + + return __create_response({ + "installations": user_installations["installations"] + }) + + +@web_server.route("/installs/installation_selection") +def installs_installation_selection(): + installation_id = request.args.get('installation_id') + + CredentialsManager.write_credentials(installation_access_token=GithubInterface.get_installation_access_token(installation_id, FileInterface.load_private_key())) + + return __create_response({}) + +@web_server.route("/github_app_installation_callback") +def github_app_installation_callback(): + ''' + TODO: fix bug, when coming back from the installation page from github to the callback with my main account (saturnin13) + the installation_id in the url is set to "undefined". find why and fix it. + ''' + installation_id = request.args.get('installation_id') + setup_action = request.args.get('setup_action') + + installation_access_token = GithubInterface.get_installation_access_token(installation_id, FileInterface.load_private_key()) + + CredentialsManager.write_credentials(installation_access_token=installation_access_token) + + installation = {} + if CredentialsManager.read_credentials()["user_access_token"]: + user_installations = GithubInterface.get_user_installations() + + for installation in user_installations["installations"]: + if installation["id"] == installation_id: + installation = installation + + return __create_response({ + "installation": installation + }) + +def __create_response(json): + response = jsonify(json) + + response.headers['Access-Control-Allow-Origin'] = '*' + return response + +def __create_option_response(): + response = Response() + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Headers'] = 'Content-Type' + return response + + + + diff --git a/backend/server/webhook_server.py b/backend/server/webhook_server.py new file mode 100644 index 0000000..70e296e --- /dev/null +++ b/backend/server/webhook_server.py @@ -0,0 +1,54 @@ +import json + +from flask import jsonify, request, abort, Blueprint + +from github_interface.interface import GithubInterface +from mongo.models import Document +from utils.constants import GITHUB_WEBHOOK_SECRET + +webhook_server = Blueprint('webhook_server', __name__,) + + +@webhook_server.route("/webhook_handler", methods=['POST']) +def webhook_handler(): + print("Webhook has been called") + signature = request.headers['X-Hub-Signature'] + body = request.get_data() + + if not GithubInterface.verify_signature(signature, body, GITHUB_WEBHOOK_SECRET): + abort(401) + + data = json.loads(request.data.decode("utf-8")) + repo = GithubInterface.get_repo(data["repository"]["full_name"]) + + for commit in data["commits"]: + ref = data["ref"] + if __is_branch_master(ref): + sha = commit["id"] + commit_files = repo.get_commit_files(sha=sha) + __update_db_ref_line_numbers(repo.full_name, commit_files) + + response = jsonify({}) + + return response + +def __is_branch_master(ref): + return ref[ref.rfind('/') + 1:] == "master" + +def __update_db_ref_line_numbers(repo_name, commit_files): + name_commit_files = [commit_file.previous_path for commit_file in commit_files] + + for document in Document.get_all(): + document_json = document.to_json() + for ref in document_json["refs"]: + + if ref["repo"] == repo_name and ref["path"] in name_commit_files: + commit_file = list(filter(lambda x: x.previous_path == ref["path"], commit_files))[0] + updated_line_range = commit_file.calculate_updated_line_range(ref["start_line"], ref["end_line"]) + Document.update_lines_ref(ref["ref_id"], updated_line_range[0], updated_line_range[1]) + + if commit_file.has_path_changed: + Document.update_path_ref(ref["ref_id"], commit_file.path) + + if commit_file.is_deleted: + Document.update_is_deleted_ref(ref["ref_id"], True) diff --git a/backend/test.py b/backend/test.py index 63c4ba7..5e072c3 100644 --- a/backend/test.py +++ b/backend/test.py @@ -1,17 +1,86 @@ +import datetime import json import sys import time +from github_interface.git.git_diff_parser import GitDiffParser + +from github import Github, GithubIntegration, BadCredentialsException + from github_interface.interface import GithubInterface from utils.json.custom_json_encoder import CustomJsonEncoder + + + +raw_patch = "@@ -1,4 +1,5 @@\n" \ +" 1\n" \ +"+1bis\n" \ +"+2\n" \ +" 3\n" \ +" 4\n" \ +"@@ -37,3 +38,4 @@ hello world\n" \ +" hello world\n" \ +" hello world\n" \ +" hello world\n" \ +"+hello world" +parser = GitDiffParser(raw_patch) +print(parser.calculate_updated_line_range(1, 6)) + + +try: + g = Github("etufwf") + g.get_rate_limit() +except BadCredentialsException as e: + print("error printed") + +# GITHUB_APP_IDENTIFIER = 33713 +# INSTALLATION_ID = 1191487 + +with open('ressources/github-private-key.pem', 'rb') as file: + private_key = file.read() + +# g2 = GitHub() +# g2.login_as_app_installation(private_key, GITHUB_APP_IDENTIFIER, INSTALLATION_ID) +# +# # print(dir(g2.repository("saturnin13", "tech-company-documentation").file_contents(""))) +# print(g2.repository("saturnin13", "tech-company-documentation").directory_contents("")) +# print(dir(g2.repository("saturnin13", "tech-company-documentation").file_contents("Makefile"))) +# print(base64.b64decode(g2.repository("saturnin13", "tech-company-documentation").file_contents("Makefile").content)) +# print(g2.all_repositories()) +# # for repo in g2.all_repositories(): +# # print(dir(repo)) + +integration = GithubIntegration(str(GITHUB_APP_IDENTIFIER), private_key) +print(integration.create_jwt()) +print(datetime.datetime.now()) +print("access token: " + str(integration.get_access_token(INSTALLATION_ID).token)) +print("access token expire time: " + str(integration.get_access_token(INSTALLATION_ID).expires_at)) + +access_token = integration.get_access_token(INSTALLATION_ID).token +access_token = "v1.31eb4345c03f4f2d0e7779c0cbaf5902ec9f3035" + +g2 = GithubInterface(access_token=access_token, is_user_access_token=False) +# root_directory = g2.get_repo("saturnin13/tech-company-documentation").root_directory +# root_directory.load_subfiles() +# print(root_directory.subfiles["backend"]) +# root_directory.subfiles["backend"].load_subfiles() +# print(root_directory.subfiles["backend"].subfiles["web_server.py"].content) +print("laaaaaaaaaaaaaaaaaaaaaa") +for repo in g2.get_repos(): + print(repo.full_name) +print(g2.get_installation(INSTALLATION_ID)) +print() + + + start = time.time() # repo_name = "saturnin13/tech-company-documentation" repo_name = "louisblin/LondonHousingForecast-Backend" # repo_name = "paulvidal/1-week-1-tool" -g = GithubInterface("39180cc3f47072520e81a31484291ea5acc5af9f") +g = GithubInterface(access_token="39180cc3f47072520e81a31484291ea5acc5af9f", is_user_access_token=True) repo = g.get_repo(repo_name) repo_root = repo.root_directory @@ -53,7 +122,7 @@ for commit_file in commit_files: print() print(commit_file.calculate_updated_line_range(3, 50)) - print(commit_file.has_path_changed()) + print(commit_file.has_path_changed) print(commit_file.previous_path) print(commit_file.path) @@ -117,3 +186,5 @@ # # #TODO check why the second part is not being taken into account by the range (fix for diff_parser.calculate_updated_line_range(2, 11)) +# GITHUB_WEBHOOK_SECRET = SatPaulDocumentation +# GITHUB_APP_IDENTIFIER = 33713 \ No newline at end of file diff --git a/backend/utils/constants.py b/backend/utils/constants.py new file mode 100644 index 0000000..e61d6ed --- /dev/null +++ b/backend/utils/constants.py @@ -0,0 +1,6 @@ +CLIENT_ID = "Iv1.82c79af55b4c6b95" +CLIENT_SECRET = "62226729b900229f67ba534a2eb54f74abeadd4b" +GITHUB_APP_IDENTIFIER = "33713" +SECRET_PASSWORD_FORGERY = "secret_password" +REDIRECT_URL_LOGIN = "http://localhost:8080/auth/github/callback" +GITHUB_WEBHOOK_SECRET = "SatPaulDocumentation" \ No newline at end of file diff --git a/backend/utils/file_interface.py b/backend/utils/file_interface.py new file mode 100644 index 0000000..be81672 --- /dev/null +++ b/backend/utils/file_interface.py @@ -0,0 +1,8 @@ +class FileInterface: + + @staticmethod + def load_private_key(): + with open('backend/ressources/github-private-key.pem', 'rb') as file: + private_key = file.read() + file.close() + return private_key \ No newline at end of file diff --git a/backend/web_server.py b/backend/web_server.py deleted file mode 100644 index fe7b8ea..0000000 --- a/backend/web_server.py +++ /dev/null @@ -1,181 +0,0 @@ -import copy -import uuid - -from flask import Flask, jsonify, request, abort, Response -from pygments import highlight -from pygments.formatters.html import HtmlFormatter -from pygments.lexers import get_lexer_for_filename -from pygments.lexers.special import TextLexer -from pygments.util import ClassNotFound - -from github_interface.interface import GithubInterface -from mongo.models import Document -from utils import code_formatter -from utils.json.custom_json_encoder import CustomJsonEncoder - -app = Flask(__name__) - -app.json_encoder = CustomJsonEncoder - -github_interface = GithubInterface("39180cc3f47072520e81a31484291ea5acc5af9f") - -@app.route("/github") -def github(): - repo = github_interface.get_repo("saturnin13/tech-company-documentation") - content = repo.get_content_at_path("website/src/main.js").content - - # Lexer to determine language - lexer = get_lexer_for_filename("website/src/main.js") - formatter = HtmlFormatter(noclasses=True, cssclass='card card-body') - result = highlight(content, lexer, formatter) - - result = '# This is awesome\n## This is also cool\n Here is some highlighted code using the library [pigments](http://pygments.org/docs/quickstart/)\n\n' \ - + result - - response = jsonify(result) - response.headers['Access-Control-Allow-Origin'] = '*' - - return response - - -@app.route("/repos") -def repos(): - # Get the repository list - repo_names = [r.full_name for r in github_interface.get_repos()] - - # Return the response - response = jsonify(repo_names) - response.headers['Access-Control-Allow-Origin'] = '*' - - return response - - -@app.route("/file") -def files(): - # Get the repository - repo_name = request.args.get('repo') - - if not repo_name: - return abort(400, "A repo should be specified") - - # repo = g.get_repo("saturnin13/tech-company-documentation") - repo = github_interface.get_repo(repo_name) - - # Get the content at path - path_arg = request.args.get('path') - path = path_arg if path_arg else "" - - repo_object = copy.deepcopy(repo.get_content_at_path(path)) - - # Syntax highlighting for file - if repo_object.type == 'file': - try: - lexer = get_lexer_for_filename(path) - except ClassNotFound: - lexer = TextLexer() # use a generic lexer if we can't find anything - - formatter = HtmlFormatter(noclasses=True, linenos='table', linespans='code-line') - repo_object.content = highlight(repo_object.content, lexer, formatter) - - # Return the response - response = jsonify(repo_object) - response.headers['Access-Control-Allow-Origin'] = '*' - - return response - - -@app.route("/save", methods=['POST', 'OPTIONS']) -def save(): - if request.method == 'OPTIONS': - response = Response() - response.headers['Access-Control-Allow-Origin'] = '*' - response.headers['Access-Control-Allow-Headers'] = 'Content-Type' - return response - - if Document.find(request.get_json().get('name')): - return abort(400, 'Document name already exists') - - doc = Document.from_json(request.get_json()) - doc.insert() - - response = Response() - response.headers['Access-Control-Allow-Origin'] = '*' - return response - - -@app.route("/docs") -def docs(): - docs = Document.get_all() - - response = jsonify([doc.to_json() for doc in docs]) - response.headers['Access-Control-Allow-Origin'] = '*' - return response - - -@app.route("/render") -def render(): - name = request.args.get('name') - - # Get the documentation doc - doc = Document.find(name) - - references = {} - - for ref in doc.references: - repo = github_interface.get_repo(ref.repo) - - content = '\n'.join(repo.get_lines_at_path(ref.path, ref.start_line, ref.end_line)) - formatted_code = code_formatter.format(ref.path, content, ref.start_line) - - references[ref.ref_id] = { - 'code': formatted_code, - 'repo': ref.repo, - 'path': ref.path, - 'startLine': ref.start_line, - 'endLine': ref.end_line, - } - - response = jsonify({ - 'name': name, - 'content': doc.content, - 'refs': references - }) - - response.headers['Access-Control-Allow-Origin'] = '*' - return response - - -# TODO: similar to render -> refactor later -@app.route("/lines") -def get_lines(): - repo = request.args.get('repo') - path = request.args.get('path') - start_line = int(request.args.get('startLine')) - end_line = int(request.args.get('endLine')) - - repository = github_interface.get_repo(repo) - content = ''.join(repository.get_content_at_path(path).content.splitlines(keepends=True)[start_line - 1: end_line]) - - try: - lexer = get_lexer_for_filename(path) - except ClassNotFound: - lexer = TextLexer() # use a generic lexer if we can't find anything - - formatter = HtmlFormatter(noclasses=True, linenos='table', linespans='code-line', linenostart=start_line) - code = highlight(content, lexer, formatter) - - response = jsonify({ - 'ref_id': str(uuid.uuid1()), # generate a unique id for the reference - 'code': code, - 'repo': repo, - 'path': path, - 'startLine': start_line, - 'endLine': end_line, - }) - - response.headers['Access-Control-Allow-Origin'] = '*' - return response - - -if __name__ == '__main__': - app.run() \ No newline at end of file diff --git a/frontend/src/App.vue b/frontend/src/App.vue index f322f4b..2d28f8c 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -7,13 +7,13 @@ Home
  • - Markdown + Markdown
  • - Browser + Browser
  • - Documents + Documents
    • diff --git a/frontend/src/router.js b/frontend/src/router.js index 9a8b545..2aaba33 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -4,6 +4,11 @@ import Home from './views/Home.vue' import Markdown from './views/Markdown.vue' import BrowserView from './views/BrowserView.vue' import Documents from './views/Documents.vue' +import Login from './views/Login.vue' +import AuthGithubCallback from "./views/AuthGithubCallback"; +import Installs from "./views/Installs"; +import AppInstallationCallbackGithub from "./views/AppInstallationCallbackGithub"; +import AuthGithub from "./views/AuthGithub"; Vue.use(Router); @@ -17,25 +22,44 @@ export default new Router({ component: Home }, { - path: '/markdown', + path: '/installs/:account_id/markdown', name: 'markdown', component: Markdown }, { - path: '/browser', + path: '/installs/:account_id/browser', name: 'browser', component: BrowserView }, { - path: '/docs', + path: '/installs/:account_id/docs', name: 'docs', component: Documents }, - // { - // path: '/markdown', - // name: 'markdown', - // // separate chunk (about.[hash].js) lazy-loaded when the route is visited. - // component: () => import(/* webpackChunkName: "about" */ './views/Markdown.vue') - // } + { + path: '/login', + name: 'login', + component: Login + }, + { + path: '/auth/github', + name: 'auth_github', + component: AuthGithub + }, + { + path: '/auth/github/callback', + name: 'auth_github_callback', + component: AuthGithubCallback + }, + { + path: '/installs', + name: 'installs', + component: Installs + }, + { + path: '/auth/github/app_installation_callback', + name: 'auth_github_app_installation_callback', + component: AppInstallationCallbackGithub + } ] }) diff --git a/frontend/src/views/AppInstallationCallbackGithub.vue b/frontend/src/views/AppInstallationCallbackGithub.vue new file mode 100644 index 0000000..8378d44 --- /dev/null +++ b/frontend/src/views/AppInstallationCallbackGithub.vue @@ -0,0 +1,20 @@ + + + diff --git a/frontend/src/views/AuthGithub.vue b/frontend/src/views/AuthGithub.vue new file mode 100644 index 0000000..cb94567 --- /dev/null +++ b/frontend/src/views/AuthGithub.vue @@ -0,0 +1,18 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/AuthGithubCallback.vue b/frontend/src/views/AuthGithubCallback.vue new file mode 100644 index 0000000..3e1a2c3 --- /dev/null +++ b/frontend/src/views/AuthGithubCallback.vue @@ -0,0 +1,23 @@ + + + diff --git a/frontend/src/views/Installs.vue b/frontend/src/views/Installs.vue new file mode 100644 index 0000000..8cc4d55 --- /dev/null +++ b/frontend/src/views/Installs.vue @@ -0,0 +1,53 @@ + + + diff --git a/frontend/src/views/Login.vue b/frontend/src/views/Login.vue new file mode 100644 index 0000000..8c190d0 --- /dev/null +++ b/frontend/src/views/Login.vue @@ -0,0 +1,5 @@ + \ No newline at end of file diff --git a/frontend/src/views/Markdown.vue b/frontend/src/views/Markdown.vue index 3c85155..f402e48 100644 --- a/frontend/src/views/Markdown.vue +++ b/frontend/src/views/Markdown.vue @@ -133,7 +133,8 @@ 'repo': content.repo, 'path': content.path, 'start_line': content.startLine, - 'end_line': content.endLine + 'end_line': content.endLine, + 'is_deleted': false }) } diff --git a/frontend/src/views/elements/browser/DirectoryBrowserDisplay.vue b/frontend/src/views/elements/browser/DirectoryBrowserDisplay.vue index 7394cbd..bf3ef52 100644 --- a/frontend/src/views/elements/browser/DirectoryBrowserDisplay.vue +++ b/frontend/src/views/elements/browser/DirectoryBrowserDisplay.vue @@ -20,8 +20,4 @@ } } } - - - \ No newline at end of file + \ No newline at end of file diff --git a/frontend/src/views/elements/browser/FileBrowserDisplay.vue b/frontend/src/views/elements/browser/FileBrowserDisplay.vue index 5802514..6af0510 100644 --- a/frontend/src/views/elements/browser/FileBrowserDisplay.vue +++ b/frontend/src/views/elements/browser/FileBrowserDisplay.vue @@ -34,8 +34,6 @@ selectLines(event) { let id = event.target.id; - console.log(event); - if (!id) { return; } diff --git a/frontend/src/views/elements/browser/RepositoryBrowserDisplay.vue b/frontend/src/views/elements/browser/RepositoryBrowserDisplay.vue index f42488f..33ff4d2 100644 --- a/frontend/src/views/elements/browser/RepositoryBrowserDisplay.vue +++ b/frontend/src/views/elements/browser/RepositoryBrowserDisplay.vue @@ -20,8 +20,4 @@ } } } - - - \ No newline at end of file + \ No newline at end of file