forked from Purple-Team-SIC/twitch-bigdata-ML-project
-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.js
137 lines (124 loc) · 2.92 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
// Required modules
const tmi = require('tmi.js');
const axios = require('axios');
const kafka = require('kafka-node');
// Categories which messages will be fetched
const categoriesIds = [
['509658'],
['26936', '509660', '509673', '1469308723', '509659'],
['488190', '498566', '499634', '29452'],
['417752', '515214', '509663'],
['509672', '509671', '509667', '272263131', '518203', '116747788'],
[
'512710',
'1614555304',
'515025',
'516575',
'32982',
'32399',
'511224',
'33214',
'491931',
'263490',
],
[
'18122',
'21779',
'29595',
'27471',
'491487',
'493597',
'490100',
'65632',
'386821',
'459931',
],
[
'102007682',
'1745202732',
'513143',
'513181',
'138585',
'11989',
'31339',
'862021340',
'31376',
'2748',
],
];
// API requests for each category
const categoriesRequests = [];
// Channels to be fetched from each category
const channels = [];
// GET channel streaming specific category on twitch
const getChannelsStreamingCategory = async (category) => {
try {
const res = await axios.get(
`https://api.twitch.tv/helix/streams?type=live&language=en&first=100&game_id=${category.join(
'&game_id='
)}`,
{
headers: {
'Client-Id': 'g4dfdnc0soguw5es28kykivzwtpdw5',
Authorization: 'Bearer wedmevykkgdf1xxx631g07dgdqlnnu',
},
}
);
return res;
} catch (e) {
console.log(e);
}
};
// API call to get all channels for each category
const getChannels = async () => {
for (let i = 0; i < categoriesIds.length; i++) {
categoriesRequests.push(
await getChannelsStreamingCategory(categoriesIds[i])
);
}
try {
const promises = await Promise.all(categoriesRequests);
promises.forEach(({ data }) => {
const res = data.data.map((a) => a.user_login);
channels.push(res);
});
} catch (err) {
console.log(err);
}
};
// Start the client that will listen for messages
const startClient = async (producer) => {
await getChannels();
const client = new tmi.Client({
connection: {
secure: true,
reconnect: true,
},
channels: channels[0],
});
client.connect();
client.on('message', (channel, tags, message, self) => {
if (tags.emotes == null && tags['message-type'] == 'chat' && !tags.mod) {
payloads = [
{
topic: 'sparkTopic',
messages: message,
timestamp: Date.now(),
},
{
topic: 'hdfsTopic',
messages: message,
timestamp: Date.now(),
},
];
producer.send(payloads, (err, data) => {
console.log(`DATA: ${JSON.stringify(data)}`);
});
}
});
};
const kafkaClient = new kafka.KafkaClient({ kafkaHost: 'localhost:9092' });
const kafkaProducer = new kafka.HighLevelProducer(kafkaClient);
kafkaProducer.on('ready', () => {
startClient(kafkaProducer);
});