Start hornet implementation #1

* init Django project
* init hornet application
* hornet models: account, member, message
* hornet client: list near and favorite members, message, set filter
* hornet management commands: account, list members, list messages, etc

.gitignore

@@ -99,3 +99,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# pycharm
+.idea/*

hornet/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

hornet/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class HornetConfig(AppConfig):
+    name = 'hornet'

hornet/client.py

@@ -0,0 +1,94 @@
+from logging import getLogger
+
+import requests
+
+from . import models
+
+logger = getLogger(__name__)
+
+
+class Client(object):
+    BASE_URL = "https://hornet.com/api/v3/"
+
+    def __init__(self, account):
+        self.session = requests.Session()
+        self.account = account
+        self._authenticated = False
+        self._check_authentication()
+
+    def _check_authentication(self):
+        if self._authenticated:
+            return
+        if self.account.token:
+            logger.debug("Set token from account")
+            self.session.headers["Authorization"] = "Hornet " + self.account.token
+            self._authenticated = True
+
+    def set_filter(self, min_age, max_age):
+        logger.info("Set filters")
+        logger.info("Age filter: %s %s", min_age, max_age)
+        age_filter = {"category": "general", "key": "age", "data": {"min": min_age, "max": max_age}}
+        filters = {"filters": [{"filter": age_filter}]}
+        url = self.BASE_URL + "filters.json"
+        response = self.session.post(url, json=filters)
+        logger.debug("Response %s", response)
+        response.raise_for_status()
+        print(response.json())
+
+    def _list_members(self, url, page_num, page_size):
+        logger.debug("Request url %s", url)
+        response = self.session.get(url, params={"page": page_num, "per_page": page_size})
+        logger.debug("Response %s", response)
+        response.raise_for_status()
+        member_list = response.json()['members']
+
+        result = []
+        for member_data in member_list:
+            member = models.Member.get(self.account, member_data['member'])
+            member.save()
+            result.append(member)
+        return result
+
+    def list_near(self, page_num, page_size):
+        logger.info("List near profiles: page number %s, page size %s", page_num, page_size)
+        return self._list_members(self.BASE_URL + "members/near.json", page_num, page_size)
+
+    def list_favorites(self, page_num, page_size):
+        logger.info("List near profiles: page number %s, page size %s", page_num, page_size)
+        return self._list_members(self.BASE_URL + "favourites/favourites.json", page_num, page_size)
+
+    def list_message(self, member):
+        logger.info("List messages with %s", member)
+        url = self.BASE_URL + "messages/" + member.network_id + "/conversation.json"
+        logger.debug("Request url %s", url)
+        response = self.session.get(url, params={"profile_id": member.network_id,
+                                                 "per_page": 1000})
+        response.raise_for_status()
+        messages_list = response.json()['messages']
+        result = []
+
+        for message_data in messages_list:
+            message = models.Message.get(member, message_data['message'])
+            message.save()
+            result.append(message)
+
+        return result
+
+    def send_message(self, member, text):
+        logger.info("Send message to %s", member)
+        params = {"recipient": member.network_id, "type": "chat", "data": text}
+        url = self.BASE_URL + "messages.json"
+        response = self.session.post(url, json=params)
+        response.raise_for_status()
+        logger.debug("Message sent")
+
+    def increment_list(self, method, limit):
+        logger.info("Increment download ")
+        member_list = []
+        page_number = 0
+        while len(member_list) < limit:
+            logger.debug("Load page: %s", page_number)
+            member_list.extend(method(self, page_number, 100))
+            logger.debug("Loaded %s members", len(member_list))
+            page_number += 1
+        return member_list

hornet/management/commands/common.py

@@ -0,0 +1,41 @@
+from logging import getLogger
+
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.utils.functional import cached_property
+
+from hornet.client import Client
+from hornet.models import Account
+
+logger = getLogger(__name__)
+
+
+class ClientCommand(BaseCommand):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(stdout=None, stderr=None, no_color=False)
+        self._account_username = None
+
+    def create_parser(self, prog_name, subcommand):
+        parser = super(ClientCommand, self).create_parser(prog_name, subcommand)
+        parser.add_argument("--account", type=int)
+        return parser
+
+    def execute(self, *args, **options):
+        self._account_username = options.pop("account", None)
+        super().execute(*args, **options)
+
+    @cached_property
+    def client(self):
+        if self._account_username:
+            account = Account.get_account(self._account_username)
+        else:
+            account = Account.objects.first()
+
+        if not account:
+            raise CommandError("Account doesn't found")
+
+        return Client(account)
+
+    def handle(self, *args, **options):
+        super(ClientCommand, self).handle(*args, **options)

hornet/management/commands/debug.py

@@ -0,0 +1,38 @@
+from django.core.management.base import BaseCommand
+from hornet.client import Client
+import pprint
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        client = Client()
+        client.set_token("Si1It2chqp4D2JI8LNa5HnBHrg3qbfpv")
+        res = []
+        page_number = 0
+        while len(res) < 2000:
+            res.extend(client.list_near(page_number, 100))
+            self.stderr.write("Loaded %s\n" % len(res))
+            page_number += 1
+        if res[0].distance is None:
+            res[0].distance = 0
+            res[0].save()
+        for idx in range(1, len(res) - 1):
+            if res[idx].distance is not None:
+                continue
+            member = res[idx]
+            print("no data at ", idx, ":", member)
+            prev_member = res[idx - 1]
+            next_member = None
+            for idx in range(idx + 1, len(res)):
+                if res[idx].distance is not None:
+                    print("found next member with distance at", idx)
+                    next_member = res[idx]
+                    break
+            if next_member:
+                print("prev", prev_member, "next", next_member, "update", member)
+                member.distance = (prev_member.distance + next_member.distance) / 2
+                member.save()
+
+        pprint.pprint(res)
+
+        self.stderr.write("members: %s\n" % len(res))

hornet/management/commands/hornet_account.py

@@ -0,0 +1,61 @@
+from logging import getLogger
+
+from django.core.management.base import BaseCommand
+
+from hornet.client import Client
+from hornet.models import Account
+
+logger = getLogger(__name__)
+
+
+CREATE = "create"
+UPDATE = "update"
+PRINT = "print"
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", choices=[CREATE, UPDATE, PRINT])
+        parser.add_argument("username")
+        parser.add_argument("--password")
+        parser.add_argument("--token")
+
+    def handle(self, username, action, *args, **kwargs):
+        account = Account.get_account(username)
+        if action == CREATE:
+            self.create(account, username, **kwargs)
+        elif action == UPDATE:
+            self.update(account, **kwargs)
+        elif action == PRINT:
+            self.print(account)
+        else:
+            raise AssertionError("Unknown action")
+
+    def create(self, account, username, token, **kwargs):
+        if account:
+            self.stderr.write("Account exists")
+            return
+        account = Account.objects.create(username=username, token=token)
+        self.stdout.write("Account created")
+        self.print(account)
+
+    def update(self, account, token, **kwargs):
+        if not account:
+            self.stderr.write("Not found")
+            return
+        account.token = token
+        account.save(update_fields=["token"])
+        self.print(account)
+
+    def print(self, account):
+        if not account:
+            self.stderr.write("Not found")
+            return
+        self.stdout.writelines([
+            " pk: %s" % account.pk,
+            " username: %s" % account.username,
+            " token: %s" % account.token,
+        ])
+
+

hornet/management/commands/hornet_list_members.py

@@ -0,0 +1,33 @@
+from logging import getLogger
+
+from hornet import models
+from hornet import utils
+from .common import ClientCommand
+
+logger = getLogger(__name__)
+
+
+class Command(ClientCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument("--limit", type=int, default=200)
+        parser.add_argument("--update-distance", action="store_true")
+        parser.add_argument("--method", choices=["near", "favorites"], default="near")
+
+    def handle(self, limit, update_distance, method, *args, **options):
+        if update_distance:
+            logger.debug("Reset distance")
+            models.Member.objects.update(distance=None)
+        if method == "near":
+            method = self.client.list_near
+        elif method == "favorites":
+            method = self.client.list_favorites
+        else:
+            raise AssertionError("Unknown method")
+        member_list = utils.increment_list(method, limit)
+        if update_distance:
+            utils.update_distance(member_list)
+
+        self.stderr.write("members: %s\n" % len(member_list))
+        for idx, member in enumerate(member_list):
+            self.stdout.write(" %s: %s" % (idx, member))

hornet/management/commands/hornet_list_messages.py

@@ -0,0 +1,24 @@
+from logging import getLogger
+
+from hornet import models
+from .common import ClientCommand
+
+logger = getLogger(__name__)
+
+
+class Command(ClientCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument("member_id", type=int)
+
+    def handle(self, member_id, *args, **kwargs):
+        try:
+            member = models.Member.objects.get(pk=member_id)
+        except models.Member.DoesNotExist:
+            self.stderr.write("Unknown member")
+            return
+
+        result = self.client.list_message(member)
+        for message in result:
+            print("  ", message)
+        self.stderr.write("Total messages: %s" % len(result))

hornet/management/commands/hornet_overview.py

@@ -0,0 +1,53 @@
+from logging import getLogger
+
+from django.utils.timezone import now
+
+from hornet import utils
+from .common import ClientCommand
+
+logger = getLogger(__name__)
+
+MIN_AGE = 6 * 3600
+
+MESSAGES = [
+    "Привет. Познакомимся?",
+    "Привет. Как дела? Чем занят?",
+    "Знакомиься будем? Привет",
+    "Давай знакомиьтся. Что ищешь тут?",
+    "Здорово. Познакомимся поближе?",
+    "Привет. Как весеннее настроение?",
+]
+
+
+class Command(ClientCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument("--limit", type=int, default=200)
+        parser.add_argument("--update-distance", action="store_true")
+        parser.add_argument("--method", choices=["near", "favorites"], default="near")
+        parser.add_argument("--distance", type=float, default=35)
+
+    def handle(self, limit, update_distance, method, distance, *args, **options):
+        member_list = utils.increment_list(self.client.list_near, 1000)
+        logger.debug("Got %s members", len(member_list))
+        utils.update_distance(member_list)
+        member_list = [i for i in member_list if i.distance is not None and i.distance <= distance]
+        logger.debug("Got %s filtered members", len(member_list))
+
+        for member in member_list:
+            logger.debug("Check member %s", member)
+            messages = sorted(self.client.list_message(member), key=lambda o: o.datetime)
+            if messages:
+                if len(messages) >= 2:
+                    logger.debug("Too much messages. Skip")
+                    continue
+                last_message = messages[-1]
+                logger.debug("Last %s", last_message)
+                age = now() - last_message.datetime
+                logger.debug("Age %s (total seconds %s)", age, age.total_seconds())
+                if age.total_seconds() < MIN_AGE:
+                    logger.debug("Skip it")
+                    continue
+            msg = utils.select_and_randomize_msg(MESSAGES)
+            logger.debug("Send message: %s", msg)
+            self.client.send_message(member, msg)