From 481d31f3410e129f3981d5153a14b7cf9643b51a Mon Sep 17 00:00:00 2001 From: Yuki Sawa Date: Wed, 8 Nov 2023 15:26:14 -0800 Subject: [PATCH] upgrade ci --- .cpa/flake8.cfg | 72 +++++++++++++++++++++ .cpa/prettier.json | 7 ++ .github/workflows/ci.yaml | 85 +++++++++++++++++-------- .gitignore | 105 ++++++++++++++++++++++++------ .pre-commit-config.yaml | 130 +++++++++++++++++++++++++++++--------- Dockerfile | 12 ++++ Makefile | 25 ++++++++ pyproject.toml | 48 ++++++++++++-- 8 files changed, 403 insertions(+), 81 deletions(-) create mode 100644 .cpa/flake8.cfg create mode 100644 .cpa/prettier.json create mode 100644 Dockerfile create mode 100644 Makefile diff --git a/.cpa/flake8.cfg b/.cpa/flake8.cfg new file mode 100644 index 0000000..6408689 --- /dev/null +++ b/.cpa/flake8.cfg @@ -0,0 +1,72 @@ +[flake8] +ignore = +# C901, # function is too complex. Ignored because max-complexity is set. + D100, + # Missing docstring in public module. + D101, + # Missing docstring in public class. + D102, + # Missing docstring in public method. + D103, + # Missing docstring in public function. + D104, + # Missing docstring in public package. + D105, + # Missing docstring in magic method. + D107, + # Missing docstring in __init__. + D205, + # 1 blank line required between summary line and description. + D400, + # First line should end with a period. + E203, + # whitespace before ':'. Conflicts with how Black formats slicing. + E231, + # missing whitespace after ',', ';', or ':'. Conflicts with Black. + E266, + # too many leading '#' for block comment. + E402, + # module level import not at top of file. + E501, + # line too long (82 > 79 characters). Ignored because max-line-length is set. + F841, + # local variable is assigned to but never used. + I100, + # Import statements are in the wrong order. + I201, + # Missing newline between import groups. + I202, + # Additional newline in a group of imports. + W503 + # line break before binary operator. This is no longer PEP 8 compliant. + +exclude = + .cache, + .coverage.*, + .env, + .git, + .github, + .gradle, + .hg, + .mypy_cache, + .pytest_cache, + .svn, + .tox, + .venv, + .vscode, + *__pycache__, + *.egg-info, + *.pyc, + build, + dist, + htmlcov.*, + +# List of application-specific import names. +application-import-names = flake8 +# Import statement format style. +import-order-style = google +# The maximum McCabe complexity allowed. +max-complexity = 18 +# The maximum allowed line length. +max-line-length = 120 +# per-file-ignores = # Per-file-ignores setting can be used to ignore specific errors in specific files. diff --git a/.cpa/prettier.json b/.cpa/prettier.json new file mode 100644 index 0000000..64f80e2 --- /dev/null +++ b/.cpa/prettier.json @@ -0,0 +1,7 @@ +{ + "bracketSpacing": true, + "singleQuote": false, + "useTabs": false, + "tabWidth": 2, + "trailingComma": "all" +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a55a9a6..3e0f536 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,52 +1,83 @@ name: CI -#################################### -# Start the job on all push and PR # -#################################### on: - pull_request: + pull_request: # Start the job on all PRs branches: [master, main] types: [synchronize, opened, reopened, ready_for_review] - push: + push: # Start the job on all main branch push branches: [master, main] jobs: - precommits: + precommit: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" + + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip- + restore-keys: | + ${{ runner.os }}-pip- + + - name: Cache shfmt binary + uses: actions/cache@v3 + with: + path: /usr/local/bin/shfmt + # key: ${{ runner.os }}-shfmt-${{ env.SHFMT_VERSION }} + key: ${{ runner.os }}-shfmt- + restore-keys: | + ${{ runner.os }}-shfmt- + + - name: Cache Pre-Commit environments + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + # key: ${{ runner.os }}-pc-${{ hashFiles('.pre-commit-config.yaml') }} + key: ${{ runner.os }}-pc- + restore-keys: | + ${{ runner.os }}-pc- - name: Install dependencies run: | - python -m pip install -r requirements-dev.txt + python -m pip install pre-commit pre-commit install + - name: Install shfmt + run: | + SHFMT_VERSION="v3.7.0" + SHFMT_BIN="shfmt_${SHFMT_VERSION}_linux_amd64" + if [[ ! -f /usr/local/bin/shfmt ]]; then + wget -O shfmt "https://github.com/mvdan/sh/releases/download/${SHFMT_VERSION}/${SHFMT_BIN}" + chmod +x shfmt + sudo mv shfmt /usr/local/bin/ + fi + sudo apt-get install shellcheck + - name: Run pre-commits + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - pre-commit run --all-files + REPO_NAME=$(echo $GITHUB_REPOSITORY | sed 's/^.*\///') + DEFAULT_BRANCH=$(curl -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY" | jq -r '.default_branch') - lintyaml: - runs-on: ubuntu-latest # Set the agent to run on + git fetch + CUR_SHA=$(git log --pretty=tformat:"%H" -n1 . | tail -n1) - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 + echo "Default branch is $DEFAULT_BRANCH" + echo "Current SHA is $CUR_SHA" - - name: Lint code - uses: docker://github/super-linter:v3.3.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEFAULT_BRANCH: main - VALIDATE_YAML: true - VALIDATE_ALL_CODEBASE: false - VALIDATE_PYTHON: false + if [[ $GITHUB_REF == "refs/heads/$DEFAULT_BRANCH" ]]; then + pre-commit run --all + else + pre-commit run --from-ref origin/$DEFAULT_BRANCH --to-ref $CUR_SHA + fi diff --git a/.gitignore b/.gitignore index e6e79af..bd3422b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +########################################################################################## +# Python +# From: https://github.com/github/gitignore/blob/main/Python.gitignore +########################################################################################## # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -14,12 +18,13 @@ dist/ downloads/ eggs/ .eggs/ -# lib/ +lib/ lib64/ parts/ sdist/ var/ wheels/ +share/python-wheels/ *.egg-info/ .installed.cfg *.egg @@ -38,14 +43,17 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ +.nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover +*.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -55,6 +63,7 @@ coverage.xml *.log local_settings.py db.sqlite3 +db.sqlite3-journal # Flask stuff: instance/ @@ -67,16 +76,49 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints -# pyenv -.python-version +# IPython +profile_default/ +ipython_config.py -# celery beat schedule file +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff celerybeat-schedule +celerybeat.pid # SageMath parsed files *.sage.py @@ -102,24 +144,49 @@ venv.bak/ # mypy .mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +########################################################################################## +# Rust +# From: https://github.com/github/gitignore/blob/main/Rust.gitignore +########################################################################################## +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ -.DS_Store -.idea - -release.sh - -.gitconfig +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock -**/config_load_check_tool +# These are backup files generated by rustfmt +**/*.rs.bk -**/tempCodeRunnerFile* +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb -**/.cache/ -**/.serverless/ -**/node_modules/ -*/.coverage -**/htmlcov +########################################################################################## +# Misc +########################################################################################## +tmp* -backend/bin/ -backend/lib/ +backend/bin +.serverless diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb9f144..09b29e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,39 +1,111 @@ default_language_version: - # Force all unspecified python hooks to run python3 python: python3 + repos: + ############################################################################# + # Misc + ############################################################################# + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-merge-conflict # Searches for merge conflict markers within files. + - id: check-added-large-files # Blocks commits that add large files. Default limit is 500kB. + # Can be configured with args, e.g., '--maxkb=1000' to change the limit. + # exclude: 'your_dir/.*' + # args: ['--maxkb=5000'] + - id: check-case-conflict # Identifies potential case-insensitive file name conflicts. + - id: check-ast # Validates the syntax of Python files. + - id: check-symlinks # Detects broken symlinks. + - id: trailing-whitespace # Removes any trailing whitespace at the end of lines. + - id: end-of-file-fixer # Ensures files end with a single newline or are empty. + + ############################################################################# + # JSON, TOML + ############################################################################# + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-json # Validates JSON files to ensure they are properly formatted and syntactically correct. + types: [json] + - id: check-toml # Checks TOML files for errors and format issues to ensure valid syntax. + types: [toml] + + ############################################################################# + # Shell + ############################################################################# + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shfmt # Formats shell scripts to a standard convention using shfmt. + - id: shellcheck # Lints shell scripts to identify syntax and usage errors, with a specified severity of 'warning'. + args: + - --severity=warning + + ############################################################################# + # Python + ############################################################################# + - repo: https://github.com/PyCQA/autoflake + rev: v2.2.1 + hooks: + - id: autoflake # Removes unused imports and unused variables from Python code. + args: + - --in-place + - --remove-all-unused-imports + - repo: https://github.com/pycqa/isort - rev: 5.6.4 + rev: 5.12.0 hooks: - - id: isort - files: "backend" - args: ["--settings-path", "setup.cfg"] + - id: isort # Sorts Python imports into sections and by alphabetical order. + args: + - --settings-path + - pyproject.toml + types: + - python + - repo: https://github.com/psf/black - rev: 22.8.0 + rev: 23.10.1 hooks: - - id: black - files: "backend" - args: ["--config", "pyproject.toml"] - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - id: black # Formats Python code to conform to the Black code style. + args: + - --config + - pyproject.toml + types: + - python + + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 hooks: - - id: flake8 - files: "backend" + - id: flake8 # Lints Python code for errors and code style issues based on PEP8. + args: + - --config=.cpa/flake8.cfg + types: + - python + + # - repo: https://github.com/astral-sh/ruff-pre-commit + # rev: v0.1.4 + # hooks: + # - id: ruff + + # - repo: https://github.com/python-poetry/poetry + # rev: '1.7.0' + # hooks: + # - id: poetry-check # Makes sure the poetry configuration does not get committed in a broken state. + # - id: poetry-lock # Ensures the poetry.lock file is up-to-date with the pyproject.toml changes. + + ############################################################################# + # CSS, Markdown, JavaScript, TypeScript, YAML style formatter + ############################################################################# - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.2.1" + rev: v3.0.3 hooks: - - id: prettier - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 # Use the ref you want to point at. Run pre-commit autoupdate after py3 - hooks: - - id: check-ast # Simply check whether files parse as valid python. - files: "backend" - - id: check-yaml - - id: check-added-large-files # Prevent giant files from being committed. Default 500kB. - - id: requirements-txt-fixer # Sorts entries in requirements.txt - - id: check-case-conflict # Check for files with names that would conflict on a case-insensitive filesystem like MacOS HFS+ or Windows FAT. - - id: trailing-whitespace - - id: check-merge-conflict # Check for files that contain merge conflict strings. - - id: check-toml - - id: check-json - - id: end-of-file-fixer # Ensure newline at eof + - id: prettier # An opinionated code formatter supporting multiple languages. + name: prettier + args: [--config, .cpa/prettier.json, --write] + types_or: + - css + - scss + - ts + - tsx + - javascript + - yaml + - markdown diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1b4b414 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.10-slim + +# Set the working directory in the container to /app +WORKDIR /app + +# Copy the current directory contents into the container at /app +COPY . . + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +CMD ["python", "./main.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5c2a614 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +.PHONY: setuppc +setuppc: + @echo "Setting up pre-commit and hooks..." + python3 -m pip install pre-commit + pre-commit install + +ifeq ($(shell uname),Darwin) + @echo "Setting up shfmt (macOS)..." + brew install shfmt + + @echo "Setting up shellcheck (macOS)..." + brew install shellcheck +else + @echo "Setting up shfmt (Linux)..." + wget -qO shfmt "https://github.com/mvdan/sh/releases/download/v3.7.0/shfmt_v3.7.0_$(shell uname -m)" + chmod +x shfmt + sudo mv shfmt /usr/local/bin/shfmt + + @echo "Setting up shellcheck (Linux)..." + sudo apt-get install shellcheck || sudo yum install shellcheck || sudo dnf install shellcheck +endif + +.PHONY: reqtxt +reqtxt: + poetry export -f requirements.txt --output requirements.txt --without-hashes diff --git a/pyproject.toml b/pyproject.toml index 81e2c9d..b0d3a44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,20 @@ -# Example configuration for Black. +[tool.poetry] +name = "" +version = "0.0.1" +description = "" +authors = [ + "My Name " +] +license = "" -# NOTE: you have to use single-quoted strings in TOML for regular expressions. -# It's the equivalent of r-strings in Python. Multiline strings are treated as -# verbose regular expressions by Black. Use [ ] to denote a significant space -# character. +[build-system] +requires = ["poetry-core>=1.7.0"] +build-backend = "poetry.core.masonry.api" [tool.black] line-length = 120 skip-string-normalization = false -target-version = ['py38'] +target-version = ['py310'] include = '\.pyi?$' exclude = ''' /( @@ -24,3 +30,33 @@ exclude = ''' | dist )/ ''' +color = true + +[tool.isort] +balanced_wrapping = true +include_trailing_comma = true +known_first_party = "" +known_third_party = [ + "boto3", # Common for AWS + "django", # Common web framework, if used + "flask", # Common web framework, if used + "jinja2", # Common templating engine + "matplotlib", # Common for plotting + "numpy", # Common for numerical operations + "pandas", # Common for data manipulation + "pendulum", # Common for date time + "pytest", # Common for testing + "requests", # Common for HTTP requests + "sqlalchemy", # Common ORM for databases +] +multi_line_output = 3 +profile = "black" +line_length = 120 + +[tool.poetry.dependencies] +python = "^3.10" + +[tool.poetry.group.dev.dependencies] +pre-commit = ">=3.5.0" +pytest = "^7.3.1" +pytest-cov = "^4.1.0"