Merge pull request #2451 from chaoss/dev

Release 0.51.0

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.50.3
+# Augur NEW Release v0.51.0
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
@@ -8,7 +8,7 @@
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.50.3
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.0
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard

augur/api/routes/__init__.py

@@ -9,4 +9,5 @@
 from .nonstandard_metrics import *
 from .pull_request_reports import *
 from .user import *
+from .dei import *
 from .util import *

augur/api/routes/dei.py

@@ -0,0 +1,134 @@
+"""
+Creates routes for DEI badging functionality
+"""
+
+import logging, subprocess, inspect
+
+from flask import request, Response, jsonify, render_template, send_file
+from pathlib import Path
+
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm.exc import NoResultFound
+
+from augur.api.util import api_key_required, ssl_required
+from augur.util.repo_load_controller import RepoLoadController
+
+from augur.application.db.models import User, ClientApplication, CollectionStatus, Repo, RepoGroup, BadgingDEI
+from augur.application.db.session import DatabaseSession
+from augur.application.config import AugurConfig
+
+from augur.tasks.util.collection_util import start_block_of_repos, get_enabled_phase_names_from_config, core_task_success_util
+from augur.tasks.start_tasks import prelim_phase, primary_repo_collect_phase
+from augur.tasks.github.util.github_task_session import GithubTaskSession
+from augur.tasks.init.redis_connection import redis_connection as redis
+from augur.tasks.github.util.util import get_repo_weight_by_issue
+
+from ..server import app, engine
+
+logger = logging.getLogger(__name__)
+Session = sessionmaker(bind=engine, autocommit=True)
+
+from augur.api.routes import AUGUR_API_VERSION
+from augur.application.db.models.augur_operations import FRONTEND_REPO_GROUP_NAME
+
+@app.route(f"/{AUGUR_API_VERSION}/dei/repo/add", methods=['POST'])
+@ssl_required
+@api_key_required
+def dei_track_repo(application: ClientApplication):
+    dei_id = request.args.get("id")
+    level = request.args.get("level")
+    repo_url = request.args.get("url")
+
+    if not (dei_id and level and repo_url):
+        return jsonify({"status": "Missing argument"}), 400
+
+    repo_url = repo_url.lower()
+
+    session = DatabaseSession(logger)
+    session.autocommit = True
+    repo: Repo = session.query(Repo).filter(Repo.repo_git==repo_url).first()
+    if repo:
+        # Making the assumption that only new repos will be added with this endpoint
+        return jsonify({"status": "Repo already exists"})
+
+    frontend_repo_group: RepoGroup = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first()
+    repo_id = Repo.insert(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="")
+    if not repo_id:
+        return jsonify({"status": "Error adding repo"})
+
+    repo = Repo.get_by_id(session, repo_id)
+    repo_git = repo.repo_git
+    pr_issue_count = get_repo_weight_by_issue(logger, repo_git)
+
+    record = {
+        "repo_id": repo_id,
+        "issue_pr_sum": pr_issue_count,
+        "core_weight": -9223372036854775808,
+        "secondary_weight": -9223372036854775808,
+        "ml_weight": -9223372036854775808
+    }
+
+    collection_status_unique = ["repo_id"]
+    session.insert_data(record, CollectionStatus, collection_status_unique, on_conflict_update=False)
+
+    record = {
+        "badging_id": dei_id,
+        "level": level,
+        "repo_id": repo_id
+    }
+
+    enabled_phase_names = get_enabled_phase_names_from_config(logger, session)
+
+    #Primary collection hook.
+    primary_enabled_phases = []
+
+    #Primary jobs
+    if prelim_phase.__name__ in enabled_phase_names:
+        primary_enabled_phases.append(prelim_phase)
+
+    primary_enabled_phases.append(primary_repo_collect_phase)
+
+    #task success is scheduled no matter what the config says.
+    def core_task_success_util_gen(repo_git):
+        return core_task_success_util.si(repo_git)
+
+    primary_enabled_phases.append(core_task_success_util_gen)
+
+    record = BadgingDEI(**record)
+    session.add(record)
+    start_block_of_repos(logger, session, [repo_url], primary_enabled_phases, "new")
+
+    session.close()
+
+    return jsonify({"status": "Success"})
+
+@app.route(f"/{AUGUR_API_VERSION}/dei/report", methods=['POST'])
+@ssl_required
+@api_key_required
+def dei_report(application: ClientApplication):
+    dei_id = request.args.get("id")
+
+    if not dei_id:
+        return jsonify({"status": "Missing argument"}), 400
+
+    session = DatabaseSession(logger)
+
+    project: BadgingDEI = session.query(BadgingDEI).filter(BadgingDEI.badging_id==dei_id).first()
+
+    if not project:
+        return jsonify({"status": "Invalid ID"})
+
+    md = render_template("dei-badging-report.j2", project=project)
+    cachePath = Path.cwd() / "augur" / "static" / "cache"
+
+    source = cachePath / f"{project.id}_badging_report.md"
+    report = cachePath / f"{project.id}_badging_report.pdf"
+    source.write_text(md)
+
+    command = f"mdpdf -o {str(report.resolve())} {str(source.resolve())}"
+    converter = subprocess.Popen(command.split())
+    converter.wait()
+
+    # TODO what goes in the report?
+
+    return send_file(report.resolve())

augur/api/routes/user.py

@@ -28,7 +28,7 @@
 from ..server import app, engine
 
 logger = logging.getLogger(__name__)
-development = get_development_flag()
+current_user: User = current_user
 Session = sessionmaker(bind=engine)
 
 from augur.api.routes import AUGUR_API_VERSION
@@ -88,7 +88,7 @@ def generate_session(application):
     # Some apps use form data instead of arguments
     code = request.args.get("code") or request.form.get("code")
     if not code:
-        return jsonify({"status": "Missing argument: code"})
+        return jsonify({"status": "Missing argument: code"}), 400
 
     grant_type = request.args.get("grant_type") or request.form.get("grant_type")
 
@@ -130,7 +130,7 @@ def refresh_session(application):
     refresh_token_str = request.args.get("refresh_token")
 
     if not refresh_token_str:
-        return jsonify({"status": "Invalid refresh token"})
+        return jsonify({"status": "Missing argument: refresh_token"}), 400
 
     if request.args.get("grant_type") != "refresh_token":
         return jsonify({"status": "Invalid grant type"})
@@ -141,8 +141,8 @@ def refresh_session(application):
         if not refresh_token:
             return jsonify({"status": "Invalid refresh token"})
 
-        if refresh_token.user_session.application == application:
-            return jsonify({"status": "Applications do not match"})
+        if refresh_token.user_session.application != application:
+            return jsonify({"status": "Invalid application"})
 
         user_session = refresh_token.user_session
         user = user_session.user
@@ -154,8 +154,10 @@ def refresh_session(application):
         session.delete(user_session)
         session.commit()
 
-    return jsonify({"status": "Validated", "refresh_token": new_refresh_token_id, "access_token": new_user_session_token, "expires": 86400})
+        response = jsonify({"status": "Validated", "refresh_token": new_refresh_token_id, "access_token": new_user_session_token, "expires": 86400})
+        response.headers["Cache-Control"] = "no-store"
 
+    return response
 
 @app.route(f"/{AUGUR_API_VERSION}/user/query", methods=['POST'])
 @ssl_required

augur/api/routes/util.py

@@ -85,10 +85,10 @@ def get_repos_in_repo_group(repo_group_id):
         FROM
             repo
             left outer join
-            (select repo_id, COUNT ( distinct commits.cmt_commit_hash ) AS commits_all_time from commits group by repo_id ) a on
+            (select * from api_get_all_repos_commits) a on
             repo.repo_id = a.repo_id
             left outer join
-            (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
+            (select * from api_get_all_repos_issues) b
             on
             repo.repo_id = b.repo_id
             left outer join 

augur/api/util.py

@@ -136,10 +136,11 @@ def wrapper(*args, **kwargs):
             session = Session()
             try:
                 kwargs["application"] = session.query(ClientApplication).filter(ClientApplication.api_key == client_token).one()
-                return fun(*args, **kwargs)
-            except NoResultFound:
-                pass
+            except NoResultFound as e:
+                return {"status": "Unauthorized client"}
 
+            return fun(*args, **kwargs)
+
         return {"status": "Unauthorized client"}
 
     return wrapper

augur/application/cli/backend.py

@@ -77,7 +77,7 @@ def start(disable_collection, development, port):
 
         worker_vmem_cap = config.get_value("Celery", 'worker_process_vmem_cap')
 
-    gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} augur.api.server:app"
+    gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} augur.api.server:app --log-file gunicorn.log"
     server = subprocess.Popen(gunicorn_command.split(" "))
 
     time.sleep(3)

augur/application/db/models/__init__.py

@@ -107,7 +107,8 @@
     Subscription,
     SubscriptionType,
     RefreshToken,
-    CollectionStatus
+    CollectionStatus,
+    BadgingDEI
 )
 
 DEFAULT_REPO_GROUP_IDS = [1, 10]

augur/application/db/models/augur_operations.py

@@ -221,6 +221,22 @@ class WorkerSettingsFacade(Base):
     comment="For future use when we move all working tables to the augur_operations schema. ",
 )
 
+class BadgingDEI(Base):
+    id = Column(Integer, primary_key=True, nullable=False)
+    badging_id = Column(Integer, nullable=False)
+    level = Column(String, nullable=False)
+
+    repo_id = Column(
+        ForeignKey("augur_data.repo.repo_id", name="user_repo_user_id_fkey"), primary_key=True, nullable=False
+    )
+
+    repo = relationship("Repo")
+
+    __tablename__ = 'dei_badging'
+    __table_args__ = (
+        {"schema": "augur_data"}
+    )
+
 
 class Config(Base):
     id = Column(SmallInteger, primary_key=True, nullable=False)
@@ -1046,6 +1062,8 @@ class CollectionStatus(Base):
     __tablename__ = "collection_status"
     __table_args__ = (
 
+        #TODO: normalize this table to have a record per repo and collection hook instead of just being per repo.
+
         #Constraint to prevent nonsensical relationship states between core_data_last_collected and core_status
         #Disallow core_data_last_collected status to not be set when the core_status column indicates data has been collected
         #Disallow core_data_last_collected status to be set when the core_status column indicates data has not been collected
@@ -1121,9 +1139,14 @@ class CollectionStatus(Base):
     facade_data_last_collected = Column(TIMESTAMP)
     facade_task_id = Column(String)
 
+    ml_status = Column(String,nullable=False, server_default=text("'Pending'"))
+    ml_data_last_collected = Column(TIMESTAMP)
+    ml_task_id = Column(String)
+
     core_weight = Column(BigInteger)
     facade_weight = Column(BigInteger)
     secondary_weight = Column(BigInteger)
+    ml_weight = Column(BigInteger)
 
     issue_pr_sum = Column(BigInteger)
     commit_sum = Column(BigInteger)
@@ -1155,7 +1178,8 @@ def insert(session, repo_id):
             "repo_id": repo_id,
             "issue_pr_sum": pr_issue_count,
             "core_weight": github_weight,
-            "secondary_weight": github_weight
+            "secondary_weight": github_weight,
+            "ml_weight": github_weight
         }
 
         result = session.insert_data(record, CollectionStatus, collection_status_unique, on_conflict_update=False)

augur/application/db/session.py

@@ -49,7 +49,7 @@ def remove_null_characters_from_list_of_dicts(data_list, fields):
 
 class DatabaseSession(Session):
 
-    def __init__(self, logger, engine=None, from_msg=None):
+    def __init__(self, logger, engine=None, from_msg=None, **kwargs):
 
         self.logger = logger
         self.engine = engine
@@ -68,7 +68,7 @@ def __init__(self, logger, engine=None, from_msg=None):
             else:
                 logger.debug(f"ENGINE CREATE")
 
-        super().__init__(self.engine)
+        super().__init__(self.engine, **kwargs)
 
     def __enter__(self):
         return self

augur/application/schema/alembic/versions/20_add_dei_badging_table.py

@@ -0,0 +1,40 @@
+"""Add DEI Badging table
+
+Revision ID: 20
+Revises: 19
+Create Date: 2023-06-15 06:02:42.082872
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '20'
+down_revision = '19'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('dei_badging',
+    sa.Column('id', sa.Integer, nullable=False, autoincrement=True),
+    sa.Column('badging_id', sa.Integer(), nullable=False),
+    sa.Column('level', sa.String(), nullable=False),
+    sa.Column('repo_id', sa.BigInteger(), nullable=False),
+    sa.ForeignKeyConstraint(['repo_id'], ['augur_data.repo.repo_id'], name='user_repo_user_id_fkey'),
+    sa.PrimaryKeyConstraint('id', 'repo_id'),
+    schema='augur_data'
+    )
+
+    op.add_column('users', sa.Column('email_verified', sa.Boolean(), server_default='false', nullable=False), schema='augur_operations')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('users', 'email_verified', schema='augur_operations')
+
+    op.drop_table('dei_badging', schema='augur_data')
+    # ### end Alembic commands ###