From 854225d52dde04b481e2f4f943bbd4e1178ad3fa Mon Sep 17 00:00:00 2001 From: WolfwithSword <12175651+WolfwithSword@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:18:49 -0300 Subject: [PATCH 1/3] add edge weighting and tooltips --- README.md | 1 + config.ini | 1 + data/streamer_connection.py | 16 ++++++++-- helpers/config.py | 4 +++ helpers/twitch_utils.py | 13 ++++---- main.py | 19 +++++++++++- templates/template.html | 59 ++++++++++++++++++++++--------------- 7 files changed, 81 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 9495400..994ca36 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ primary_channel: `channel_name` for your primary channel(s). Can be a comma sepa blacklisted_users: `twitchname_1,twitchname_2` comma separated list of channels names to ignore in the network generation. Sponsor/Corporate accounts are a good option here as it will help cut down on users! +weighted_edges: `true/false` whether to thicken lines/edges between users based on how many times they've collab'd #### [DATA] diff --git a/config.ini b/config.ini index 0f2d6c5..e22f332 100644 --- a/config.ini +++ b/config.ini @@ -2,6 +2,7 @@ use_images=true primary_channel=REPLACEME,REPLACEMEORREMOVEME blacklisted_users=twitchname_1,twitchname_2 +weighted_edges=false [DATA] max_depth=7 diff --git a/data/streamer_connection.py b/data/streamer_connection.py index 69367a3..6140be5 100644 --- a/data/streamer_connection.py +++ b/data/streamer_connection.py @@ -9,14 +9,26 @@ def __init__(self, twitch_user: TwitchUser): self.children = [] self.processed = False self.color = "blue" + self.collab_counts = dict() def add_child(self, child: Streamer): if child not in self.children: self.children.append(child) + def add_collab(self, collaborator: Streamer, was_tagged=True): + val = 1 + if not was_tagged: + val = 0 + if collaborator.name not in self.collab_counts: + self.collab_counts[collaborator.name] = val + else: + self.collab_counts[collaborator.name] += val + if was_tagged: + self.add_child(collaborator) + @property def size(self): - return len(self.children) + return len(self.collab_counts) @property def done(self): @@ -26,4 +38,4 @@ def done(self): def node_color(self): if self.processed: return self.color - return "red" \ No newline at end of file + return "red" diff --git a/helpers/config.py b/helpers/config.py index ba5d4a2..68346fb 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -75,3 +75,7 @@ def blacklisted_channelnames(self) -> list: @property def cache_enabled(self) -> bool: return self.getboolean(section='CACHE', option='enabled', fallback=False) + + @property + def weighted_edges(self) -> bool: + return self.getboolean(section='DISPLAY', option='weighted_edges', fallback=False) diff --git a/helpers/twitch_utils.py b/helpers/twitch_utils.py index 121445e..1651e1d 100644 --- a/helpers/twitch_utils.py +++ b/helpers/twitch_utils.py @@ -99,13 +99,16 @@ async def find_connections_from_videos(self, videos: list[Video], u = await self.get_user_by_name(username=n) if u: child = StreamerConnection(u) - user.add_child(child) - child.add_child(user) # Bidirectional enforcement + user.add_collab(child) + child.add_collab(user, was_tagged=False) users[child.name] = child elif n not in [x.name.strip() for x in user.children]: - user.add_child(users[n]) - if user not in users[n].children: # Bidirectional enforcement - users[n].add_child(user) # Bidirectional enforcement + user.add_collab(users[n]) + if user not in users[n].children: + users[n].add_collab(user, was_tagged=False) + elif n in users: + # Just increment collab counter + user.add_collab(users[n]) if len(users) >= self.config.max_connections: break if user.size >= self.config.max_children: diff --git a/main.py b/main.py index 23f8647..37c8440 100644 --- a/main.py +++ b/main.py @@ -142,10 +142,27 @@ async def twitch_run(): url=f"https://twitch.tv/{user.name}", channel_name=user.name, connections=len(user.children), border=user.node_color) + weighted_edges = config.weighted_edges for u in users: user = users[u] for child in user.children: - G.add_edge(user.name, child.name) + if user.name == child.name: + title = (f"{user.name} tagged themselves {user.collab_counts.get(child.name, 0)} " + f"time{'s' if user.collab_counts.get(child.name, 0) != 1 else ''}") + weight = 1 + else: + user_tag_child_count = user.collab_counts.get(child.name, 0) + child_tag_user_count = child.collab_counts.get(user.name, 0) + title = (f"{user.name} tagged {child.name} {user_tag_child_count} " + f"time{'s' if user_tag_child_count != 1 else ''}" + f"
{child.name} tagged {user.name} {child_tag_user_count} " + f"time{'s' if child_tag_user_count != 1 else ''}") + weight = min(max(1, + max(user.collab_counts.get(child.name, 1), child.collab_counts.get(user.name, 1)) + * 0.6), 10) + if not weighted_edges: + weight = 1 + G.add_edge(user.name, child.name, title=title, weight=weight, parent=user.name, child=child.name) net = Network(notebook=False, height="1500px", width="100%", bgcolor="#222222", diff --git a/templates/template.html b/templates/template.html index e24ac52..a048537 100644 --- a/templates/template.html +++ b/templates/template.html @@ -508,31 +508,42 @@

{{heading}}

// showing the popup function showPopup(nodeId) { - // get the data from the vis.DataSet - var nodeData = nodes.get([nodeId]); - popup.innerHTML = nodeData[0].title; - // get the position of the node - var posCanvas = network.getPositions([nodeId])[nodeId]; - - // get the bounding box of the node - var boundingBox = network.getBoundingBox(nodeId); - - //position tooltip: - posCanvas.x = posCanvas.x + 0.5 * (boundingBox.right - boundingBox.left); - - // convert coordinates to the DOM space - var posDOM = network.canvasToDOM(posCanvas); - - // Give it an offset - posDOM.x += 10; - posDOM.y -= 20; - - // show and place the tooltip. - popup.style.display = 'block'; - popup.style.top = posDOM.y + 'px'; - popup.style.left = posDOM.x + 'px'; - } + // get the data from the vis.DataSet + var nodeData = nodes.get(nodeId); + // get the position of the node + var posCanvas = network.getPositions([nodeId])[nodeId]; + + if (!nodeData) { + var edgeData = edges.get(nodeId); + var poses = network.getPositions([edgeData.from, edgeData.to]); + var middle_x = (poses[edgeData.to].x - poses[edgeData.from].x) * 0.5; + var middle_y = (poses[edgeData.to].y - poses[edgeData.from].y) * 0.5; + posCanvas = poses[edgeData.from]; + posCanvas.x = posCanvas.x + middle_x; + posCanvas.y = posCanvas.y + middle_y; + + popup.innerHTML = edgeData.title; + } else { + popup.innerHTML = nodeData.title; + // get the bounding box of the node + var boundingBox = network.getBoundingBox(nodeId); + posCanvas.x = posCanvas.x + 0.5 * (boundingBox.right - boundingBox.left); + posCanvas.y = posCanvas.y + 0.5 * (boundingBox.top - boundingBox.bottom); + }; + + // convert coordinates to the DOM space + var posDOM = network.canvasToDOM(posCanvas); + + // Give it an offset + posDOM.x += 10; + posDOM.y -= 20; + + // show and place the tooltip. + popup.style.display = 'block'; + popup.style.top = posDOM.y + 'px'; + popup.style.left = posDOM.x + 'px'; + } {% endif %} From 166e4b83870906eb7061775c04f451618a300fc1 Mon Sep 17 00:00:00 2001 From: WolfwithSword <12175651+WolfwithSword@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:12:44 -0300 Subject: [PATCH 2/3] update readme --- README.md | 58 +++++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 994ca36..19795ff 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,9 @@ This program is still quite experimental and is mainly a proof-of-concept as it - Blue nodes indicate all channels that have been *fully* parsed - Red nodes indicate channel nodes that have been partially parsed, usually by running into limits -You can drag around nodes and hover over them for some more details. +You can drag around nodes and edges, and hover over them for some more details. -Note: No handling for rate limiting is implemented yet as this is a POC. -Due to time it takes to fetch data and parse, when this is single-threaded it should be fine without hitting rate limits as it is sequentially fetching data. - -No errors from Twitch API are handled, or rather, only config input errors are handled. Good luck. +Not all errors from Twitch API are handled. Good luck. Once you have your config setup, run `main.py` and when it is done (it will log depth progress), a file called `output.html` and folder `lib` will be created in the same directory. Open this in a web browser to view the Collab Network. @@ -41,7 +38,6 @@ Extract the zip to its own folder and make sure it has the executable, templates Configure the config.ini as per below and you're good to go. As for version updating, update whenever you feel like it by downloading a new portable version. - # Setup ### Environment @@ -49,53 +45,57 @@ Configure the config.ini as per below and you're good to go. As for version upda This was built using Python 3.10. See `requirements.txt` for dependencies ### Config -You will need to configure the `config.ini` accordingly. +You will need to configure the `config.ini` accordingly. Alternatively, you can make multiple and use the CLI parameters to run different configs. -#### [DISPLAY] -use_images: `true` if you want nodes to use profile pictures. `false` for coloured dots. - -primary_channel: `channel_name` for your primary channel(s). Can be a comma separated list of multiple channels to mark as primaries +#### [DISPLAY] -blacklisted_users: `twitchname_1,twitchname_2` comma separated list of channels names to ignore in the network generation. Sponsor/Corporate accounts are a good option here as it will help cut down on users! +| Setting | Type/Default | Description | +|-------------------|:---------------------------:|-------------------------------------------------------------------------------------------------------------------| +| use_images | boolean (`true`) | `true` for nodes to use profile pictures.
`false` for coloured dots | +| primary_channel | string(s) (comma-separated) | One or more channel names to use as primary starting channels | +| blacklisted_users | string(s) (comma-separated) | Comma separated list of channels to ignore completely.
Useful to add sponsor/corporate/company accounts here. | +| weighted_edges | boolean (`false`) | Whether or not to thicken lines/edges between users, based on number of times they've collaborated | -weighted_edges: `true/false` whether to thicken lines/edges between users based on how many times they've collab'd #### [DATA] -max_depth: `7` Max number of outward channels to look at before stopping +| Setting | Type/Default | Description | +|--------------|:------------:|-------------------------------------------------------------------------------------------| +| max_depth | int (`7`) | Max number for depth of outward channels to look at before stopping | +| max_users | int (`500`) | Max number of channels/users to look at before stopping | +| max_vods | int (`100`) | Max number of public vods on a channel to scan. Starts at latest first. Hard limit of 100 | +| max_children | int (`60`) | Max number of children a node can have before it stops processing more for *that* node | -max_users: `500` Max number of channels/users to look at before stopping - -max_vods: `100` Max number of vods on a channel to scan through. Starts at most recent. Hard limit of 100. - -max_children: `60` Max number of children a node can have before it stops processing more on *that* node #### [TWITCH] See [Twitch Developer Docs](https://dev.twitch.tv/docs/api/get-started/) on how to get your id/secret -client_id: `your_dev_app_client_id` +| Setting | Type/Default | Description | +|---------------|:------------:|------------------------------| +| client_id | string | Twitch Dev App client id | +| client_secret | string | Twitch Dev App client secret | -client_secret: `you_dev_app_client_secret` #### [CONCURRENCY] This program supports parallelism / concurrency for user processing either in API requests or cache fetching. -enabled: `true/false` enable parallel processing concurrency - -max_concurrency: `12` max number of concurrent processes to run. Recommend 5-20. If you hit rate-limiting from twitch API, it will pause until the rate opens back up. +| Setting | Type/Default | Description | +|-----------------|:----------------:|--------------------------------------------------------------------------------------------------------------------------------------------------| +| enabled | boolean (`true`) | Enable/Disable parallel processing for API calls to Twitch | +| max_concurrency | int (`12`) | Max number of concurrent processes. Recommend 5-20.
If you hit Twitch's rate limits, it will wait until your limit resets before continuing. | #### [CACHE] This program supports file/disk based caching. Since this program is used to generate an output after running and is not run as a live service, a disk based cache is more useful than in-memory cache, as now API results can persist in between sessions. -enabled: `true/false` enable local disk caching for twitch API results. - -user_expiry_s: `3600` number of seconds to keep user API results from twitch before expiring. This can be a higher number without affecting much. - -vodlist_expiry_s: `600` number of seconds to keep list of user's vods with tagged users from twitch API before expiring. Can be long, but if a new public vod goes up, it won't be picked up until this expires +| Setting | Type/Default | Description | +|------------------|:----------------:|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| enabled | boolean (`true`) | Enable/Disable local disk caching for Twitch API results | +| user_expiry_s | int (`3600`) | Number of Seconds to cache API results for Twitch Users.
This is generally okay to have really high, as User data almost never changes. | +| vodlist_expiry_s | int (`600`) | Number of seconds to cache API results for Twitch Vod titles.
Recommend not too long, as if a streamer goes live before this expires, it will not get the newest vod title until the expiry time lapses. | ### CLI Parameters From 8431fc8ebf43e05ffa58bc14bad9ecf5ecfe7f0c Mon Sep 17 00:00:00 2001 From: WolfwithSword <12175651+WolfwithSword@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:20:19 -0300 Subject: [PATCH 3/3] update readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19795ff..59c4a24 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ This program is still quite experimental and is mainly a proof-of-concept as it - Green nodes indicate the primary channel(s) node and neighbourhood - Blue nodes indicate all channels that have been *fully* parsed - Red nodes indicate channel nodes that have been partially parsed, usually by running into limits +- Size of node is dictated by number of connections +- If configured, thickness of edges are dictated by number of collabs between two streamers You can drag around nodes and edges, and hover over them for some more details. @@ -30,12 +32,12 @@ The html file will only work if the `lib` folder is present! In that case, you can download a portable executable of this program. -Click on the [latest action here](https://github.com/WolfwithSword/TwitchCollabNetwork/actions/workflows/build.yml?query=branch%3Amain+is%3Asuccess) and download the latest build for your operating system. -Check that the tag/branch is "main" for latest/nightly. Alternatively, check on the right side of the screen for [versioned releases](https://github.com/WolfwithSword/TwitchCollabNetwork/releases/latest)! +Check on the right side of the screen for [versioned releases](https://github.com/WolfwithSword/TwitchCollabNetwork/releases/latest)! +Alternatively, click on the [latest action here](https://github.com/WolfwithSword/TwitchCollabNetwork/actions/workflows/build.yml?query=branch%3Amain+is%3Asuccess) for the latest dev/nightly release! Extract the zip to its own folder and make sure it has the executable, templates folder and config.ini. -Configure the config.ini as per below and you're good to go. As for version updating, update whenever you feel like it by downloading a new portable version. +Configure the config.ini as per below and you're good to go. As for version updating, update whenever you feel like it by downloading a new portable version and overwrite the application and template folders. For config, migrate your config manually in case new settings were added. # Setup