Skip to content

Commit

Permalink
docs: new documentation and diagrams about shared attacks, python ref…
Browse files Browse the repository at this point in the history
…actor
  • Loading branch information
domysh committed Nov 6, 2024
1 parent ae5d5a9 commit d1a8952
Show file tree
Hide file tree
Showing 46 changed files with 462 additions and 190 deletions.
5 changes: 5 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"python.analysis.extraPaths": [
"./client"
]
}
24 changes: 16 additions & 8 deletions backend/app.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,36 @@
import uvicorn
import os, asyncio, env
import os
import asyncio
import env
import time
import jwt
import socketio
import traceback
from asyncpg import UniqueViolationError, ForeignKeyViolationError
from fastapi import FastAPI, HTTPException, Depends, APIRouter, Request
from fastapi.responses import JSONResponse
from fastapi.exceptions import RequestValidationError
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
from contextlib import asynccontextmanager
import time, jwt
from fastapi.middleware.cors import CORSMiddleware
from submitter import run_submitter_daemon
from stats import run_stats_daemon
from skio import run_skio_daemon
from utils import *
from env import DEBUG, CORS_ALLOW, JWT_ALGORITHM, EXPLOIT_SOURCES_DIR
from fastapi.responses import FileResponse
from models.all import *
from utils import datetime_now, load_routers
from typing import Dict
from pydantic import ValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.middleware.base import BaseHTTPMiddleware
from db import connect_db, close_db, init_db, APP_SECRET, SERVER_ID, DBSession, Service, Team, Submitter, sqla, redis_conn, redis_channels
from skio import sio_server
import socketio
from utils.query import get_messages_array
from models.response import MessageResponse, MessageResponseInvalidError, ResponseStatus
from typing import Any
from models.config import Configuration, SetupStatus, StatusAPI
from utils import json_like
from utils import crypto

os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.makedirs(EXPLOIT_SOURCES_DIR, exist_ok=True)
Expand Down Expand Up @@ -106,7 +113,7 @@ async def unique_violation_error_handler(request, exc: UniqueViolationError):
raise HTTPException(400, error_str)

@app.exception_handler(ForeignKeyViolationError)
async def unique_violation_error_handler(request, exc: ForeignKeyViolationError):
async def foreign_key_violation_error_handler(request, exc: ForeignKeyViolationError):
error_str = "Some references to other datastructures are not valid"
try:
if exc.detail:
Expand Down Expand Up @@ -177,7 +184,8 @@ async def login_api(form: OAuth2PasswordRequestForm = Depends()):
async def get_status(db: DBSession, loggined: bool|None = Depends(is_loggined)):
""" This will return the application status, and the configuration if the user is allowed to see it """
config = await Configuration.get_from_db()
if config.PASSWORD_HASH: config.PASSWORD_HASH = "********"
if config.PASSWORD_HASH:
config.PASSWORD_HASH = "********"
messages = await get_messages_array()

teams_response = None
Expand Down Expand Up @@ -225,7 +233,7 @@ async def set_status(data: Dict[str, str|int|None], db: DBSession):
)).one_or_none()
if sub_count == 0:
raise HTTPException(400, "Submitter not found")
if key == "PASSWORD_HASH" and not data[key] is None:
if key == "PASSWORD_HASH" and data[key] is not None:
if len(str(data[key])) < 8:
raise HTTPException(400, "Password too short (at least 8 chars)")
data[key] = crypto.hash(data[key])
Expand Down
11 changes: 6 additions & 5 deletions backend/db.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
import env, secrets
import env
import secrets
import asyncio
import sqla
import time
import sqlalchemy.exc
from pydantic import BaseModel
from typing import Dict, Any, Annotated, List
from typing import Union, Callable
from uuid import UUID, uuid4
from env import RESET_DB_DANGEROUS
from hashlib import sha256
from models.enums import *
from pydantic import BeforeValidator
from utils import *
import asyncio
from fastapi import Depends
from contextlib import asynccontextmanager
from sqlalchemy.ext.asyncio import create_async_engine
Expand All @@ -18,9 +19,9 @@
from sqlmodel import Field, SQLModel, Relationship
from sqlalchemy.dialects.postgresql import JSONB
from datetime import datetime
import sqla
import redis.asyncio as redis
from env import DEBUG
from models.enums import Language, AttackExecutionStatus, FlagStatus

def datetime_now_sql(index:bool = False, now:bool = True) -> sqla.Column:
return sqla.Column(sqla.DateTime(timezone=True), server_default=sqla.func.now() if now else None, index=index)
Expand Down
2 changes: 1 addition & 1 deletion backend/env.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

VERSION = "{{VERSION_PLACEHOLDER}}" if not "{" in "{{VERSION_PLACEHOLDER}}" else "0.0.0"
VERSION = "{{VERSION_PLACEHOLDER}}" if "{" not in "{{VERSION_PLACEHOLDER}}" else "0.0.0"
DEBUG = os.getenv("DEBUG", "").lower() in ["true", "1", "t"]
CORS_ALLOW = os.getenv("CORS_ALLOW", "").lower() in ["true", "1", "t"]
RESET_DB_DANGEROUS = os.getenv("RESET_DB_DANGEROUS", "").lower() in ["true", "1", "t"]
Expand Down
2 changes: 1 addition & 1 deletion backend/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from models.response import *
from models.response import * # noqa: F403
10 changes: 0 additions & 10 deletions backend/models/all.py

This file was deleted.

7 changes: 4 additions & 3 deletions backend/models/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
from functools import cache
from pydantic import NonNegativeInt, PositiveInt
from models.submitter import SubmitterDTO
import re, env
import re
import env
from models.teams import TeamDTO
from models.service import ServiceDTO
from models.response import *
from models.response import MessageInfo
from typing import List
from models.enums import AttackMode, SetupStatus
from db import SubmitterID, Env, dbtransaction, sqla, AttackExecution, AttackExecutionStatus
Expand Down Expand Up @@ -109,7 +110,7 @@ async def __get_times(self):
from utils import datetime_now
now = datetime_now()
start_time = self.START_TIME
end_time = self.END_TIME if not self.END_TIME is None and self.END_TIME > now else None
end_time = self.END_TIME if self.END_TIME is not None and self.END_TIME > now else None
done_query = sqla.select(AttackExecution.received_at).where(AttackExecution.status == AttackExecutionStatus.done)
async with dbtransaction() as db:
if not start_time:
Expand Down
3 changes: 2 additions & 1 deletion backend/models/exploit.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from db import ExploitID, Language, ExploitStatus
from db import ExploitID, Language
from db import ServiceID, ClientID, UnHashedClientID, TeamID, ExploitSourceID, AttackGroupID
from pydantic import BaseModel, AwareDatetime, Field, AliasChoices
from models.enums import AttackExecutionStatus
from utils import json_like
from models.enums import ExploitStatus

###-- Exploit Models --###

Expand Down
2 changes: 0 additions & 2 deletions backend/models/flags.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
from pydantic import BaseModel
from models.enums import FlagStatus, AttackExecutionStatus
from pydantic import AwareDatetime
from db import FlagID, AttackExecutionID, TeamID, ExploitID, ClientID, ExploitSourceID
from utils import json_like
from pydantic import BaseModel, Field, AliasChoices

class FlagDTOSmall(BaseModel):
Expand Down
2 changes: 1 addition & 1 deletion backend/models/submitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

###-- Submitter Models --###

type SubmitterKargs = Dict[str, Dict[str, Any]]
SubmitterKargs = Dict[str, Dict[str, Any]]

class SubmitterDTO(BaseModel):
id: SubmitterID
Expand Down
11 changes: 7 additions & 4 deletions backend/routes/clients.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
from models.client import *
from models.response import *
from models.client import ClientDTO, ClientAddForm, ClientEditForm
from models.response import MessageResponse
from typing import List
from fastapi import APIRouter, HTTPException
from utils import *
from db import Client, DBSession, sqla, MANUAL_CLIENT_ID, client_id_hashing, check_client_id_hashing, redis_channels, redis_conn
from utils import json_like
from db import Client, DBSession, sqla, MANUAL_CLIENT_ID
from db import client_id_hashing, check_client_id_hashing
from db import redis_channels, redis_conn
from db import ClientID, UnHashedClientID

router = APIRouter(prefix="/clients", tags=["Clients"])

Expand Down
23 changes: 15 additions & 8 deletions backend/routes/exploits.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
from models.exploit import *
from models.response import *
from models.enums import *
from models.config import *
from utils import *
from typing import List, Annotated
import os
import tarfile
import tempfile
import shutil
import pickle
import asyncio
from models.exploit import ExploitDTO, ExploitAddForm, ExploitEditForm, ManualSubmitForm
from models.exploit import ExploitSourcePushForm, ExploitSourceDTO, ExploitSubmitForm
from models.response import MessageResponse, MessageResponseInvalidError
from models.enums import ExploitStatus, AttackExecutionStatus
from models.config import Configuration
from utils import json_like, extract_values_by_regex, datetime_now
from typing import List, Annotated, Tuple
from fastapi import APIRouter, HTTPException, UploadFile, File, Form
from typing import Dict
from env import EXPLOIT_SOURCES_DIR
import tarfile, tempfile, shutil, pickle, asyncio
from fastapi.responses import FileResponse
from exploitfarm.utils.config import ExploitConfig, check_exploit_config_exists
from exploitfarm.utils import calc_hash
from db import Exploit, DBSession, AttackExecution, sqla, ExploitSource, Flag, MANUAL_CLIENT_ID, Service, redis_channels, redis_conn
from utils.query import get_exploits_with_latest_attack, get_exploit_status
from db import ExploitID, ExploitSourceID, UnHashedClientID

router = APIRouter(prefix="/exploits", tags=["Exploits"])

Expand Down Expand Up @@ -158,7 +165,7 @@ async def exploit_stopped(exploit_id: ExploitID, db: DBSession):
return { "message": f"Exploit {exploit_id} stopped" }

@router.post("/submit", response_model=MessageResponse[Dict[str, int]])
async def exploit_submit(data: ManualSubmitForm, db: DBSession):
async def exploit_submit_manual(data: ManualSubmitForm, db: DBSession):
config = await Configuration.get_from_db()

flags = (data.flags if data.flags else [])
Expand Down
14 changes: 4 additions & 10 deletions backend/routes/flags.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@

from models.response import *
from utils import *
from models.flags import *
from models.config import *
from models.flags import FlagDTO, AttackExecutionDTO, FlagStats
from models.enums import FlagStatus, AttackExecutionStatus
from fastapi import APIRouter
from fastapi_pagination import Page, add_pagination
from fastapi_pagination.ext.sqlalchemy import paginate
from typing import Any, Generic
from fastapi import Query
from pydantic import BaseModel
from fastapi_pagination.bases import AbstractParams, RawParams

from fastapi_pagination.bases import AbstractParams, RawParams, BasePage
from math import ceil
from typing import Any, Generic, Optional, Sequence, TypeVar
from fastapi import Query
from pydantic import BaseModel
from fastapi_pagination.bases import AbstractParams, BasePage, RawParams
from fastapi_pagination.types import GreaterEqualOne, GreaterEqualZero
from fastapi_pagination.utils import create_pydantic_model
from utils.query import get_stats
from db import UnHashedClientID, Flag, sqla, AttackExecution, DBSession
from db import FlagID, TeamID, ExploitID, AttackExecutionID, ClientID
from stats import complete_stats
from sqlalchemy.orm import selectinload

Expand Down
8 changes: 4 additions & 4 deletions backend/routes/services.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from models.service import *
from models.response import *
from utils import *
from typing import List
from fastapi import APIRouter, HTTPException
from db import Service, DBSession, sqla, redis_conn, redis_channels
from db import Service, DBSession, sqla, redis_conn, redis_channels, ServiceID
from models.service import ServiceDTO, ServiceAddForm, ServiceEditForm
from models.response import MessageResponse
from utils import json_like

router = APIRouter(prefix="/services", tags=["Services"])

Expand Down
16 changes: 10 additions & 6 deletions backend/routes/submitters.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from models.submitter import *
from models.response import *
from models.config import *
from typing import List
from fastapi import APIRouter, HTTPException
from utils import *
import re
from db import Submitter, DBSession, redis_conn, redis_channels
from models.submitter import SubmitterDTO, SubmitterAddForm, SubmitterEditForm, SubmitterInfoForm, SubmitterKargs
from models.response import MessageResponse, MessageResponseInvalidError
from models.config import Configuration
from typing import Dict, Any
from utils import json_like, extract_submit, has_submit_signature, get_additional_args, type_check_annotation
from db import SubmitterID, sqla

router = APIRouter(prefix="/submitters", tags=["Submitters"])

Expand All @@ -18,7 +20,8 @@ async def new_submitter(data: SubmitterAddForm, db: DBSession):
raise HTTPException(400, error)

valid_sig, msg = has_submit_signature(submit_function)
if not valid_sig: raise HTTPException(400, msg)
if not valid_sig:
raise HTTPException(400, msg)

kargs = get_additional_args(submit_function)

Expand Down Expand Up @@ -83,7 +86,8 @@ async def update_submitter(submitter_id: SubmitterID, data: SubmitterEditForm, d
raise HTTPException(400, error)

valid_sig, msg = has_submit_signature(submit_function)
if not valid_sig: raise HTTPException(400, msg)
if not valid_sig:
raise HTTPException(400, msg)
kargs = get_additional_args(submit_function)

#Setting old values
Expand Down
7 changes: 3 additions & 4 deletions backend/routes/teams.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
from models.teams import *
from models.response import *
from models.config import *
from typing import List
from fastapi import APIRouter
from utils import *
import asyncio
from db import Team, DBSession, sqla, TeamID, redis_channels, redis_conn
from models.teams import TeamDTO, TeamAddForm, TeamEditForm
from utils import json_like
from models.response import MessageResponse

router = APIRouter(prefix="/teams", tags=["Teams"])

Expand Down
11 changes: 8 additions & 3 deletions backend/skio.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
import socketio, asyncio, uvloop, traceback, logging
import socketio
import asyncio
import uvloop
import traceback
import logging
from env import DEBUG
from multiprocessing import Process
from db import close_db, redis_conn, REDIS_CHANNEL_LIST, connect_db, dbtransaction, redis_channels
from utils.query import get_exploits_with_latest_attack, detailed_exploit_status
from models.enums import *
from models.config import Configuration
from models.enums import ExploitStatus

class StopLoop(Exception): pass
class StopLoop(Exception):
pass

redis_mgr = socketio.AsyncRedisManager(
url="redis://localhost:6379/0" if DEBUG else "redis://redis:6379/0",
Expand Down
4 changes: 2 additions & 2 deletions backend/sqla.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

#Mix of SQL Alchemy and PostgresSQL dialect feature

from sqlalchemy import *
from sqlalchemy.dialects.postgresql import *
from sqlalchemy import * # noqa: F403
from sqlalchemy.dialects.postgresql import * # noqa: F403
Loading

0 comments on commit d1a8952

Please sign in to comment.