fix: support changes to woolies & for 2Up

.env.example

@@ -1,3 +1,2 @@
-WOOLIES_CLIENT_ID=
 WOOLIES_TOKEN=
 UP_TOKEN=

README.md

@@ -42,7 +42,7 @@ customers with ✨e-receipts✨, which is more than I can say for their competit
 1. Head to [Up Banking's API](https://developer.up.com.au/#welcome) page & grab your personal API token
 2. Login to [Woolworth's Everyday Rewards](https://www.woolworthsrewards.com.au/#login) site & navigate around with
    dev-tools monitoring network traffic. Filter network traffic with `api.woolworthsrewards.com.au` & find any request
-   that has `client_id` & `authorization` headers.
+   that has `authorization` header.
    <p align="center">
     <img src="/imgs/headers.jpg" />
    </p>
@@ -52,7 +52,6 @@ customers with ✨e-receipts✨, which is more than I can say for their competit
 3. Copy `.env.example` to `.env` & place those three tokens inside:
 
 ```
-WOOLIES_CLIENT_ID=cXDN...
 WOOLIES_TOKEN=8h41...
 UP_TOKEN=up:yeah:1234abcd...
 ```
@@ -93,13 +92,19 @@ suggest the feature through support chat in-app 🙏
       that many more banks had begun supporting Open Banking than when I last checked.
     * Unfortunately, despite the title _"Consumer Data Rights"_, the process of authenticating myself with these CDR
       data holders for my _own_ consumer data is a mystery to me. If you know, then please reach out to me.
+    * [Update] I've learnt that the support/access I'm looking for fell outside the scope of CDR, and banks have no
+      obligation to support it. I would have to hold out for Data Holders or Recipients to provide.
 * 👩‍💼 Talk to someone about Woolworths' API
     * I tried reaching out to Woolworths to talk about their API: EverdayRewards support, Quantium (the tech subsidiary
       managing the program), even cold-messaged people on LinkedIn associate with WooliesX. No luck.
+    * [Update] A login endpoint was shared to me via the repo's issues. It worked like a charm, allowing user/pass
+      flows. However, it suddenly started returning 403s & I have yet to find a solution.
 * ⚡ Talk to someone about Up Bank's smart receipts
     * A friend pointed out on [The Tree of Up](https://up.com.au/tree/) a leaf call _smart receipts_ & the existing
       integration with AfterPay. It would be interesting to hear how it was implemented, & if this proof-of-concept
       shares any similarities.
+    * [Update] Dom, Co-founder of Up, gave some insight about Up
+      Bank's [smart receipts](https://twitter.com/dompym/status/1418792235559235589) integration with AfterPay
 * 👫 Support 2Up
     * Please read [help wanted](#help-wanted) on how you can help push for API support of 2Up.
 * ⚖ Interpret item weights

changelog.md

@@ -0,0 +1,19 @@
+## v1.1.0 - 20/02/2022
+
+- Added changelog 📑
+- ~~Added Woolworth's email:password login via env. vars. or CLI~~
+    - Nevermind, the woolies' login endpoint shared via [#1](https://github.com/MattTimms/up_woolies/issues/1) has begun
+      rejecting requests & instead returns 403 forbbiden
+- Added use of Up API's category-filter for off-loading some filtering compute to them
+- Added python rich library for prettier print-outs
+- Added scaffolding for accessing 2Up data
+- Updated README
+- Fixed missing/new requirement for `User-Agent` header for Woolworth's API
+- Fixed default Up spending account name from `Up Account` to `Spending` as per
+  💕 [2Up Support](https://github.com/up-banking/api/issues/31#issuecomment-1008441619) update
+- Fixed indefinite requests with default timeout adapter on request sessions
+- Fixed missing dependency versions
+
+## v1.0.0 - 24/07/2021
+
+- ⚡ initial release

requirements.txt

@@ -1,4 +1,5 @@
-python-dotenv
-requests
-prance[osv]
-pydantic
+python-dotenv==0.17.1
+requests==2.25.1
+rich==11.2.0
+prance[osv]==0.21.2
+pydantic==1.8.2

src/up_woolies/main.py

@@ -1,19 +1,25 @@
 import datetime
 import json
-from pprint import pprint
+from rich import print
 
 import up
 import woolies
 
 
+up_account = up.SpendingAccount()
+# two_up_account = up.TwoUpAccount()
+
+
 def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction) -> up.Transaction:
     """ Returns corresponding Up Bank transaction from a Woolies transaction"""
+    _up_category = 'groceries'
     _up_description = 'Woolworths'  # How Up manages human-readable merchant Id for Woolworths
 
     # Request transactions within window of the Woolies transaction
     transaction_datetime = woolies_transaction.transaction_date
     for up_transactions in up_account.get_transactions(until=transaction_datetime + datetime.timedelta(seconds=10),
-                                                       since=transaction_datetime - datetime.timedelta(seconds=10)):
+                                                       since=transaction_datetime - datetime.timedelta(seconds=10),
+                                                       category=_up_category):
         for up_transaction in up_transactions:
             # Validate transactions match
             is_merchant_woolies = up_transaction.description == _up_description
@@ -26,9 +32,7 @@ def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction)
         raise FileNotFoundError("could not find corresponding transaction with up bank")
 
 
-if __name__ == '__main__':
-
-    up_account = up.SpendingAccount()
+def example():
 
     for transactions in woolies.list_transactions():
         for woolies_transaction in transactions:
@@ -50,4 +54,8 @@ def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction)
             woolies_receipt = woolies_transaction.get_receipt()
 
             # Print it but make it pretty
-            pprint({'date': up_transaction.createdAt.astimezone().isoformat('T'), **json.loads(woolies_receipt.json())})
+            print({'date': up_transaction.createdAt.astimezone().isoformat('T'), **json.loads(woolies_receipt.json())})
+
+
+if __name__ == '__main__':
+    example()

src/up_woolies/up.py

@@ -9,7 +9,7 @@
 from prance import ResolvingParser
 from pydantic import BaseModel, Extra, UUID4
 
-from utils import parse_money
+from utils import DefaultTimeoutAdapter, parse_money
 
 # Get token from environment variables
 for fp in ['../../.env', '.env']:
@@ -18,6 +18,7 @@
 # Define endpoint & headers
 endpoint = "https://api.up.com.au/api/v1/"
 session = requests.session()
+session.mount('https://', DefaultTimeoutAdapter(timeout=5))
 session.headers.update({
     "Authorization": f"Bearer {os.environ['UP_TOKEN']}"
 })
@@ -64,27 +65,38 @@ def value(self) -> Decimal:
 class Account:
     """ Base account class """
 
-    def __init__(self, name: str):
+    def __init__(self, *,
+                 display_name: str,
+                 account_type: Literal['TRANSACTIONAL', 'SAVER'] = None,
+                 ownership_type: Literal['INDIVIDUAL', 'JOINT'] = None):
+        attributes = {k: v for k, v in {
+            'displayName': display_name,
+            'accountType': account_type,
+            'ownershipType': ownership_type,
+        }.items() if v is not None}
+
         # Find account details by name
         for account in list_accounts():
-            if name in account['attributes']['displayName']:
+            if attributes.items() <= account['attributes'].items():
                 break
         else:
-            raise ValueError(f"could not find account {name=}")
+            raise ValueError(f"could not find account matching {attributes=}")
 
         self.account = account
         self.transaction_url = account['relationships']['transactions']['links']['related']
 
     def get_transactions(self,
                          page_size: int = 10,
                          since: datetime = None,
-                         until: datetime = None) -> Generator[List[Transaction], None, None]:
+                         until: datetime = None,
+                         category: str = None) -> Generator[List[Transaction], None, None]:
         """ Yields list of transactions based off input filters """
         response = session.get(url=self.transaction_url,
                                params={
                                    'page[size]': page_size,
                                    'filter[since]': since.astimezone().isoformat('T') if since is not None else since,
                                    'filter[until]': until.astimezone().isoformat('T') if until is not None else until,
+                                   'filter[category]': category
                                }).json()
         yield [Transaction.from_response(transaction) for transaction in response['data']]
 
@@ -95,10 +107,13 @@ def get_transactions(self,
 
 
 class SpendingAccount(Account):
-    name = "Up Account"
+    def __init__(self):
+        super().__init__(display_name='Spending', account_type='TRANSACTIONAL', ownership_type='INDIVIDUAL')
+
 
+class TwoUpAccount(Account):
     def __init__(self):
-        super().__init__(name=self.name)
+        super().__init__(display_name='2Up Spending', account_type='TRANSACTIONAL', ownership_type='JOINT')
 
 
 #

src/up_woolies/utils.py

@@ -1,6 +1,19 @@
 from decimal import Decimal
 from re import sub
 
+from requests.adapters import HTTPAdapter
+from requests import PreparedRequest, Response
+
 
 def parse_money(money_str: str) -> Decimal:
     return Decimal(sub(r'[^\d.]', '', money_str))
+
+
+class DefaultTimeoutAdapter(HTTPAdapter):
+    def __init__(self, *args, timeout: float, **kwargs):
+        self.timeout = timeout
+        super().__init__(*args, **kwargs)
+
+    def send(self, request: PreparedRequest, **kwargs) -> Response:
+        kwargs['timeout'] = kwargs.get('timeout') or self.timeout
+        return super().send(request, **kwargs)

src/up_woolies/woolies.py

@@ -1,5 +1,6 @@
 import os
 import re
+import warnings
 from datetime import datetime
 from decimal import Decimal
 from typing import Dict, List, Any, Generator, Optional
@@ -8,25 +9,77 @@
 import requests
 from dotenv import load_dotenv
 from pydantic import BaseModel, Extra, condecimal, PositiveInt
+from rich.console import Console
+from rich.prompt import Prompt
 
-from utils import parse_money
+from utils import DefaultTimeoutAdapter, parse_money
 
 # Get token from environment variables
 for fp in ['../../.env', '.env']:
     load_dotenv(dotenv_path=fp)
 
 # Define endpoint & headers
-endpoint = "https://api.woolworthsrewards.com.au/wx/v1/"
+endpoint = "https://api.woolworthsrewards.com.au/wx/"
 session = requests.session()
+session.mount('https://', DefaultTimeoutAdapter(timeout=5))
 session.headers.update({
-    'client_id': os.environ['WOOLIES_CLIENT_ID'],
-    'Authorization': f"Bearer {os.environ['WOOLIES_TOKEN']}"
+    'client_id': '8h41mMOiDULmlLT28xKSv5ITpp3XBRvH',  # some universal client API ID key
+    'User-Agent': 'up_woolies'  # some User-Agent
 })
 session.hooks = {
     'response': lambda r, *args, **kwargs: r.raise_for_status()
 }
 
 
+def __init():
+    if (email := os.getenv('WOOLIES_EMAIL')) is not None and (password := os.getenv('WOOLIES_PASS')):
+        auth = Auth.login(email, password)  # TODO implement token refresh
+    elif (token := os.getenv('WOOLIES_TOKEN')) is not None:
+        warnings.warn("WOOLIES_TOKEN is deprecated, use WOOLIES_[EMAIL|PASS] instead", DeprecationWarning)
+        session.headers.update({'Authorization': f"Bearer {token}"})
+        return
+    else:
+        auth = Auth.login_cli()  # TODO implement token refresh
+    session.headers.update({'Authorization': f"Bearer {auth.bearer}"})
+
+
+class Auth(BaseModel):
+    bearer: str
+    refresh: str
+    bearerExpiredInSeconds: int
+    refreshExpiredInSeconds: int
+    passwordResetRequired: bool
+
+    @classmethod
+    def login(cls, email: str, password: str):
+        url = urljoin(endpoint, 'v2/security/login/rewards')
+        body = {'username': email, 'password': password}  # email/pass
+        res = session.post(url=url, json=body)
+        return cls.parse_obj(res.json()['data'])
+
+    @classmethod
+    def login_cli(cls):
+        Console().print('Woolworths Login')
+        email = Prompt.ask("Email")
+        password = Prompt.ask("Password", password=True)
+        return cls.login(email, password)  # TODO retry bad pass
+
+    def refresh_token(self):
+        url = urljoin(endpoint, 'v2/security/refreshLogin')
+        body = {'refresh_token': self.refresh}
+        res = session.post(url=url, json=body)
+
+        _auth = self.parse_obj(res.json()['data'])
+        for attr in self.__annotations__.keys():
+            setattr(self, attr, getattr(_auth, attr))
+        return self
+
+
+# N.B. new login options are disabled as explain in changelog
+# __init()
+session.headers.update({'Authorization': f"Bearer {os.environ['WOOLIES_TOKEN']}"})
+
+
 class Purchase(BaseModel, extra=Extra.ignore):
     """ Dataclass of an unique purchased item """
     description: str
@@ -134,8 +187,8 @@ def value(self) -> Decimal:
 
 
 def list_transactions(page: int = 0) -> Generator[List[Transaction], None, None]:
-    """ Yields list of Transactions for global Woolies account """
-    url = urljoin(endpoint, 'rewards/member/ereceipts/transactions/list')
+    """ Yields list ("page") of Transactions for global Woolies account """
+    url = urljoin(endpoint, 'v1/rewards/member/ereceipts/transactions/list')
     while True:
         page += 1  # Endpoint indexes at 1
         response = session.get(url=url, params={"page": page})
@@ -146,7 +199,7 @@ def list_transactions(page: int = 0) -> Generator[List[Transaction], None, None]
 
 
 def get_receipt(receipt_key: str) -> ReceiptDetails:
-    url = urljoin(endpoint, 'rewards/member/ereceipts/transactions/details')
+    url = urljoin(endpoint, 'v1/rewards/member/ereceipts/transactions/details')
     body = {"receiptKey": receipt_key}
     response = session.post(url=url, json=body)
     return ReceiptDetails.from_raw(response.json()['data'])