Rename credit transaction fields to reflect retirement context

fly.prod.toml

@@ -5,8 +5,8 @@ kill_timeout = 5
 primary_region = "dfw"
 
 [build]
-builder = "paketobuildpacks/builder-jammy-full:latest"
-buildpacks = ["gcr.io/paketo-buildpacks/python"]
+builder = "heroku/builder:24"
+buildpacks = ["heroku/buildpack-python:0.17.0"]
 
 [[vm]]
   size = "performance-2x"
@@ -22,8 +22,12 @@ PORT = "8000"
 [deploy]
 release_command = "bash -l release.sh"
 
+[processes]
+  web = "gunicorn -w $OFFSETS_DB_WEB_CONCURRENCY -t 120 -k uvicorn.workers.UvicornWorker offsets_db_api.main:app --config gunicorn_config.py --access-logfile '-' --error-logfile '-'"
+
 
 [[services]]
+processes = ["web"]
 protocol = "tcp"
 internal_port = 8000
 force_https = true

fly.staging.toml

@@ -5,8 +5,8 @@ kill_timeout = 5
 primary_region = "dfw"
 
 [build]
-builder = "paketobuildpacks/builder-jammy-full:latest"
-buildpacks = ["gcr.io/paketo-buildpacks/python"]
+builder = "heroku/builder:24"
+buildpacks = ["heroku/buildpack-python:0.17.0"]
 
 
 [[vm]]
@@ -24,7 +24,11 @@ PORT = "8000"
 release_command = "bash -l release.sh"
 
 
+[processes]
+  web = "gunicorn -w $OFFSETS_DB_WEB_CONCURRENCY -t 120 -k uvicorn.workers.UvicornWorker offsets_db_api.main:app --config gunicorn_config.py --access-logfile '-' --error-logfile '-'"
+
 [[services]]
+processes = ["web"]
 protocol = "tcp"
 internal_port = 8000
 force_https = true

migrations/versions/6c863cafdf94_add_beneficiary_data_to_credit_model.py → migrations/versions/8b97dd22cafd_rename_credit_transaction_fields_to_.py

@@ -1,8 +1,8 @@
-"""add beneficiary data to credit model
+"""Rename credit transaction fields to reflect retirement context
 
-Revision ID: 6c863cafdf94
+Revision ID: 8b97dd22cafd
 Revises:
-Create Date: 2024-08-28 16:25:27.798958
+Create Date: 2024-09-05 14:30:52.228849
 
 """
 
@@ -12,7 +12,7 @@
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
-revision = '6c863cafdf94'
+revision = '8b97dd22cafd'
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -91,10 +91,10 @@ def upgrade() -> None:
         sa.Column('vintage', sa.Integer(), nullable=True),
         sa.Column('transaction_date', sa.Date(), nullable=True),
         sa.Column('transaction_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-        sa.Column('account', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-        sa.Column('beneficiary', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-        sa.Column('reason', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-        sa.Column('note', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('retirement_account', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('retirement_beneficiary', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('retirement_reason', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('retirement_note', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
         sa.Column('id', sa.Integer(), nullable=False),
         sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
         sa.ForeignKeyConstraint(
@@ -104,11 +104,29 @@ def upgrade() -> None:
         sa.PrimaryKeyConstraint('id'),
     )
     op.create_index(op.f('ix_credit_project_id'), 'credit', ['project_id'], unique=False)
+    op.create_index(
+        op.f('ix_credit_retirement_account'), 'credit', ['retirement_account'], unique=False
+    )
+    op.create_index(
+        op.f('ix_credit_retirement_beneficiary'), 'credit', ['retirement_beneficiary'], unique=False
+    )
+    op.create_index(op.f('ix_credit_retirement_note'), 'credit', ['retirement_note'], unique=False)
+    op.create_index(
+        op.f('ix_credit_retirement_reason'), 'credit', ['retirement_reason'], unique=False
+    )
+    op.create_index(
+        op.f('ix_credit_transaction_date'), 'credit', ['transaction_date'], unique=False
+    )
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_credit_transaction_date'), table_name='credit')
+    op.drop_index(op.f('ix_credit_retirement_reason'), table_name='credit')
+    op.drop_index(op.f('ix_credit_retirement_note'), table_name='credit')
+    op.drop_index(op.f('ix_credit_retirement_beneficiary'), table_name='credit')
+    op.drop_index(op.f('ix_credit_retirement_account'), table_name='credit')
     op.drop_index(op.f('ix_credit_project_id'), table_name='credit')
     op.drop_table('credit')
     op.drop_table('clipproject')

offsets_db_api/models.py

@@ -35,8 +35,8 @@ class ProjectBase(SQLModel):
     )
     status: str | None
     country: str | None
-    listed_at: datetime.date | None = pydantic.Field(
-        description='Date project was listed', strict=False
+    listed_at: datetime.date | None = Field(
+        description='Date project was listed',
     )
     is_compliance: bool | None = Field(description='Whether project is compliance project')
     retired: int | None = Field(
@@ -45,11 +45,11 @@ class ProjectBase(SQLModel):
     issued: int | None = Field(
         description='Total of issued credits', default=0, sa_column=Column(BigInteger())
     )
-    first_issuance_at: datetime.date | None = pydantic.Field(
-        description='Date of first issuance of credits', strict=False
+    first_issuance_at: datetime.date | None = Field(
+        description='Date of first issuance of credits',
     )
-    first_retirement_at: datetime.date | None = pydantic.Field(
-        description='Date of first retirement of credits', strict=False
+    first_retirement_at: datetime.date | None = Field(
+        description='Date of first retirement of credits',
     )
     project_url: str | None = Field(description='URL to project details')
 
@@ -67,7 +67,7 @@ class Project(ProjectBase, table=True):
 
 
 class ClipBase(SQLModel):
-    date: datetime.date = pydantic.Field(description='Date the clip was published', strict=False)
+    date: datetime.date = Field(description='Date the clip was published')
     title: str | None = Field(description='Title of the clip')
     url: str | None = Field(description='URL to the clip')
     source: str | None = Field(description='Source of the clip')
@@ -115,14 +115,14 @@ class ProjectWithClips(ProjectBase):
 class CreditBase(SQLModel):
     quantity: int = Field(description='Number of credits', sa_column=Column(BigInteger()))
     vintage: int | None = Field(description='Vintage year of credits')
-    transaction_date: datetime.date | None = pydantic.Field(
-        description='Date of transaction', strict=False
-    )
+    transaction_date: datetime.date | None = Field(description='Date of transaction', index=True)
     transaction_type: str | None = Field(description='Type of transaction')
-    account: str | None = Field(description='Account used for the transaction')
-    beneficiary: str | None = Field(description='Beneficiary of credits')
-    reason: str | None = Field(description='Reason for transaction')
-    note: str | None = Field(description='Note')
+    retirement_account: str | None = Field(
+        description='Account used for the transaction', index=True
+    )
+    retirement_beneficiary: str | None = Field(description='Beneficiary of credits', index=True)
+    retirement_reason: str | None = Field(description='Reason for transaction', index=True)
+    retirement_note: str | None = Field(description='Note', index=True)
 
 
 class Credit(CreditBase, table=True):

release.sh

@@ -6,8 +6,6 @@ set -e
 echo "Python Location: $(which python)"
 echo "Python Version: $(python --version)"
 
-echo "List of Python packages:"
-python -m pip list
 
 # Run database migrations
 echo "Running database migrations..."

requirements.txt

@@ -5,7 +5,7 @@ fastapi==0.109.1
 fastparquet
 gunicorn
 httpx
-offsets-db-data>=2024.8.0
+offsets-db-data>=2024.9.1
 pandas>=1.5.3
 psycopg2-binary==2.9.9
 pydantic-settings>=2.1

runtime.txt

@@ -0,0 +1 @@
+python-3.12.5

tests/conftest.py

@@ -50,19 +50,19 @@ def wait_for_file_processing(test_app: TestClient, file_ids: list[str], timeout:
 def setup_post(test_app: TestClient):
     payload: list[dict[str, str]] = [
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/credits-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/credits-augmented.parquet',
             'category': 'credits',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/projects-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/projects-augmented.parquet',
             'category': 'projects',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/curated-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/curated-clips.parquet',
             'category': 'clips',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-27/weekly-summary-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-03/weekly-summary-clips.parquet',
             'category': 'clips',
         },
     ]

tests/test_files.py

@@ -27,19 +27,19 @@ def test_submit_bad_file(test_app: TestClient, url: str, category: str):
 def file_urls():
     return [
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/credits-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/credits-augmented.parquet',
             'category': 'credits',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/projects-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/projects-augmented.parquet',
             'category': 'projects',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-28/curated-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-05/curated-clips.parquet',
             'category': 'clips',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-08-27/weekly-summary-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-09-03/weekly-summary-clips.parquet',
             'category': 'clips',
         },
     ]

update_database.py

@@ -68,19 +68,19 @@ def post_data_to_environment(*, env: str, bucket: str) -> None:
     else:
         files = [
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-08-28/credits-augmented.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-09-05/credits-augmented.parquet',
                 'category': 'credits',
             },
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-08-28/projects-augmented.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-09-05/projects-augmented.parquet',
                 'category': 'projects',
             },
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-08-28/curated-clips.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-09-05/curated-clips.parquet',
                 'category': 'clips',
             },
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-08-27/weekly-summary-clips.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-09-03/weekly-summary-clips.parquet',
                 'category': 'clips',
             },
         ]