diff --git a/README.md b/README.md index 3fd7df87..901fc1d2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# ScummVM File Integrity Check (GSoC 2023) +# ScummVM File Integrity Check (GSoC 2024) -This repository contains the server-side code for the upcoming file integrity check for game datafiles. This repository is part of the Google Summer of Code 2023 program. +This repository contains the server-side code for the upcoming file integrity check for game datafiles. This repository is part of the Google Summer of Code 2024 program. This website needs a `mysql_config.json` in the root to run, in the form: diff --git a/apache2-config/gamesdb.sev.zone.conf b/apache2-config/gamesdb.sev.zone.conf index 9578a31b..8b37f5be 100644 --- a/apache2-config/gamesdb.sev.zone.conf +++ b/apache2-config/gamesdb.sev.zone.conf @@ -2,22 +2,14 @@ ServerName gamesdb.sev.zone ServerAlias www.gamesdb.sev.zone ServerAdmin webmaster@localhost - DocumentRoot /var/www/vhosts.d/gamesdb.sev.zone/htdocs/ - ErrorLog /var/www/vhosts.d/gamesdb.sev.zone/logs/error.log - CustomLog /var/www/vhosts.d/gamesdb.sev.zone/logs/access.log combined - - php_admin_value open_basedir "/var/www/vhosts.d/gamesdb.sev.zone/" + CustomLog ${APACHE_LOG_DIR}/integrity-access.log combined + ErrorLog ${APACHE_LOG_DIR}/integrity-error.log + DocumentRoot /home/ubuntu/projects/python/scummvm-sites + WSGIDaemonProcess scummvm-sites user=www-data group=www-data threads=5 + WSGIScriptAlias / /home/ubuntu/projects/python/scummvm-sites/app.wsgi + + + Require all granted - - Order allow,deny - Deny from all - - - Order allow,deny - Deny from all - - - Order allow,deny - Deny from all - + diff --git a/bin/dat_parser.php b/bin/dat_parser.php deleted file mode 100644 index 92cd8485..00000000 --- a/bin/dat_parser.php +++ /dev/null @@ -1,167 +0,0 @@ -"; - // print_r($header); - // print_r($game_data); - // print_r($resources); - // echo ""; - - return array($header, $game_data, $resources, $dat_filepath); -} - -// Process command line args -if ($index = array_search("--upload", $argv)) { - foreach (array_slice($argv, $index + 1) as $filepath) { - if ($filepath == "--match") - continue; - - db_insert(parse_dat($filepath)); - } -} - -if (in_array("--match", $argv)) { - populate_matching_games(); -} - -?> - diff --git a/bin/schema.php b/bin/schema.php deleted file mode 100644 index 108f16dd..00000000 --- a/bin/schema.php +++ /dev/null @@ -1,246 +0,0 @@ -connect_errno) { - die("Connect failed: " . $conn->connect_error); -} - -// Create database -$sql = "CREATE DATABASE IF NOT EXISTS " . $dbname; -if ($conn->query($sql) === TRUE) { - echo "Database created successfully\n"; -} -else { - echo "Error creating database: " . $conn->error; - exit(); -} - -$conn->query("USE " . $dbname); - - -///////////////////////// CREATE TABLES ///////////////////////// - -// Create engine table -$table = "CREATE TABLE IF NOT EXISTS engine ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(200), - engineid VARCHAR(100) NOT NULL -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'engine' created successfully\n"; -} -else { - echo "Error creating 'engine' table: " . $conn->error; -} - -// Create game table -$table = "CREATE TABLE IF NOT EXISTS game ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(200), - engine INT NOT NULL, - gameid VARCHAR(100) NOT NULL, - extra VARCHAR(200), - platform VARCHAR(30), - language VARCHAR(10), - FOREIGN KEY (engine) REFERENCES engine(id) -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'game' created successfully\n"; -} -else { - echo "Error creating 'game' table: " . $conn->error; -} - -// Create fileset table -$table = "CREATE TABLE IF NOT EXISTS fileset ( - id INT AUTO_INCREMENT PRIMARY KEY, - game INT, - status VARCHAR(20), - src VARCHAR(20), - `key` VARCHAR(64), - `megakey` VARCHAR(64), - `delete` BOOLEAN DEFAULT FALSE NOT NULL, - `timestamp` TIMESTAMP NOT NULL, - detection_size INT, - FOREIGN KEY (game) REFERENCES game(id) -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'fileset' created successfully\n"; -} -else { - echo "Error creating 'fileset' table: " . $conn->error; -} - -// Create file table -$table = "CREATE TABLE IF NOT EXISTS file ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(200) NOT NULL, - size BIGINT NOT NULL, - checksum VARCHAR(64) NOT NULL, - fileset INT NOT NULL, - detection BOOLEAN NOT NULL, - FOREIGN KEY (fileset) REFERENCES fileset(id) ON DELETE CASCADE -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'file' created successfully\n"; -} -else { - echo "Error creating 'file' table: " . $conn->error; -} - -// Create filechecksum table -$table = "CREATE TABLE IF NOT EXISTS filechecksum ( - id INT AUTO_INCREMENT PRIMARY KEY, - file INT NOT NULL, - checksize VARCHAR(10) NOT NULL, - checktype VARCHAR(10) NOT NULL, - checksum VARCHAR(64) NOT NULL, - FOREIGN KEY (file) REFERENCES file(id) ON DELETE CASCADE -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'filechecksum' created successfully\n"; -} -else { - echo "Error creating 'filechecksum' table: " . $conn->error; -} - -// Create queue table -$table = "CREATE TABLE IF NOT EXISTS queue ( - id INT AUTO_INCREMENT PRIMARY KEY, - time TIMESTAMP NOT NULL, - notes varchar(300), - fileset INT, - userid INT NOT NULL, - commit VARCHAR(64) NOT NULL, - FOREIGN KEY (fileset) REFERENCES fileset(id) -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'queue' created successfully\n"; -} -else { - echo "Error creating 'queue' table: " . $conn->error; -} - -// Create log table -$table = "CREATE TABLE IF NOT EXISTS log ( - id INT AUTO_INCREMENT PRIMARY KEY, - `timestamp` TIMESTAMP NOT NULL, - category VARCHAR(100) NOT NULL, - user VARCHAR(100) NOT NULL, - `text` varchar(300) -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'log' created successfully\n"; -} -else { - echo "Error creating 'log' table: " . $conn->error; -} - -// Create history table -$table = "CREATE TABLE IF NOT EXISTS history ( - id INT AUTO_INCREMENT PRIMARY KEY, - `timestamp` TIMESTAMP NOT NULL, - fileset INT NOT NULL, - oldfileset INT NOT NULL, - log INT -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'history' created successfully\n"; -} -else { - echo "Error creating 'history' table: " . $conn->error; -} - -// Create transactions table -$table = "CREATE TABLE IF NOT EXISTS transactions ( - id INT AUTO_INCREMENT PRIMARY KEY, - `transaction` INT NOT NULL, - fileset INT NOT NULL -)"; - -if ($conn->query($table) === TRUE) { - echo "Table 'transactions' created successfully\n"; -} -else { - echo "Error creating 'transactions' table: " . $conn->error; -} - - -///////////////////////// CREATE INDEX ///////////////////////// - -// Create indices for fast data retrieval -// PK and FK are automatically indexed in InnoDB, so they are not included -$index = "CREATE INDEX detection ON file (detection)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'file.detection'\n"; -} -else { - echo "Error creating index for 'file.detection': " . $conn->error; -} - -$index = "CREATE INDEX checksum ON filechecksum (checksum)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'filechecksum.checksum'\n"; -} -else { - echo "Error creating index for 'filechecksum.checksum': " . $conn->error; -} - -$index = "CREATE INDEX engineid ON engine (engineid)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'engine.engineid'\n"; -} -else { - echo "Error creating index for 'engine.engineid': " . $conn->error; -} - -$index = "CREATE INDEX fileset_key ON fileset (`key`)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'fileset.key'\n"; -} -else { - echo "Error creating index for 'fileset.key': " . $conn->error; -} - -$index = "CREATE INDEX status ON fileset (status)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'fileset.status'\n"; -} -else { - echo "Error creating index for 'fileset.status': " . $conn->error; -} - -$index = "CREATE INDEX fileset ON history (fileset)"; - -if ($conn->query($index) === TRUE) { - echo "Created index for 'history.fileset'\n"; -} -else { - echo "Error creating index for 'history.fileset': " . $conn->error; -} - -$conn->close(); -?> - diff --git a/bin/seeds.php b/bin/seeds.php deleted file mode 100644 index 2c3d75f3..00000000 --- a/bin/seeds.php +++ /dev/null @@ -1,75 +0,0 @@ -connect_errno) { - die("Connect failed: " . $conn->connect_error); -} - -$conn->query("USE " . $dbname); - - -///////////////////////// INSERT VALUES ///////////////////////// - -$query = "INSERT INTO engine (name, engineid) -VALUES ('Drascula', '1')"; -$conn->query($query); -$conn->query("SET @engine_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO game (name, engine, gameid) -VALUES ('Drascula: The Vampire Strikes Back', @engine_last, '1')"; -$conn->query($query); -$conn->query("SET @game_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO file (name, size, checksum) -VALUES ('Packet.001', '32847563', 'fac946707f07d51696a02c00cc182078')"; -$conn->query($query); -$conn->query("SET @file_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO fileset (game, file, status, `key`) -VALUES (@game_last, @file_last, 0, 'fac946707f07d51696a02c00cc182078')"; -$conn->query($query); -$conn->query("SET @fileset_last = LAST_INSERT_ID()"); - -// Checksize: 0 (full checksum) -$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum) -VALUES (@file_last, '0', 'md5', 'fac946707f07d51696a02c00cc182078')"; -$conn->query($query); -$conn->query("SET @filechecksum_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO fileset_detection (fileset, checksum) -VALUES (@fileset_last, @filechecksum_last)"; -$conn->query($query); - -// Checksize: 5000B -$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum) -VALUES (@file_last, '5000', 'md5', 'c6a8697396e213a18472542d5f547cb4')"; -$conn->query($query); -$conn->query("SET @filechecksum_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO fileset_detection (fileset, checksum) -VALUES (@fileset_last, @filechecksum_last)"; -$conn->query($query); - -// Checksize: 10000B -$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum) -VALUES (@file_last, '10000', 'md5', '695f4152f02b8fa4c1374a0ed04cf996')"; -$conn->query($query); -$conn->query("SET @filechecksum_last = LAST_INSERT_ID()"); - -$query = "INSERT INTO fileset_detection (fileset, checksum) -VALUES (@fileset_last, @filechecksum_last)"; -$conn->query($query); - - -$conn->close(); -?> - diff --git a/clear.py b/clear.py new file mode 100644 index 00000000..707914c8 --- /dev/null +++ b/clear.py @@ -0,0 +1,58 @@ +""" +This script deletes all data from the tables in the database and resets auto-increment counters. +Using it when testing the data insertion. +""" + +import pymysql +import json +import os + +def truncate_all_tables(conn): + tables = ["filechecksum", "queue", "history", "transactions", "file", "fileset", "game", "engine", "log"] + cursor = conn.cursor() + + # Disable foreign key checks + cursor.execute("SET FOREIGN_KEY_CHECKS = 0") + + for table in tables: + try: + cursor.execute(f"TRUNCATE TABLE `{table}`") + print(f"Table '{table}' truncated successfully") + except pymysql.Error as err: + print(f"Error truncating table '{table}': {err}") + + # Enable foreign key checks + cursor.execute("SET FOREIGN_KEY_CHECKS = 1") + +if __name__ == "__main__": + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + servername = mysql_cred["servername"] + username = mysql_cred["username"] + password = mysql_cred["password"] + dbname = mysql_cred["dbname"] + + # Create connection + conn = pymysql.connect( + host=servername, + user=username, + password=password, + db=dbname, # Specify the database to use + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + autocommit=True + ) + + # Check connection + if conn is None: + print("Error connecting to MySQL") + exit(1) + + # Truncate all tables + truncate_all_tables(conn) + + # Close connection + conn.close() \ No newline at end of file diff --git a/compute_hash.py b/compute_hash.py index db3b793b..e5e723fa 100644 --- a/compute_hash.py +++ b/compute_hash.py @@ -79,7 +79,7 @@ def escape_string(s: str) -> str: for char in s: if char == "\x81": new_name += "\x81\x79" - elif char in '/":*|\\?%<>\x7f' or ord(char) < 0x20: + elif char in '/":*|\\?%<>\x7f' or ord(char) < 0x20 or (ord(char) & 0x80): new_name += "\x81" + chr(0x80 + ord(char)) else: new_name += char diff --git a/dat_parser.py b/dat_parser.py new file mode 100644 index 00000000..11a57b2a --- /dev/null +++ b/dat_parser.py @@ -0,0 +1,141 @@ +import re +import os +import sys +from db_functions import db_insert, populate_matching_games, match_fileset +import argparse + +def remove_quotes(string): + # Remove quotes from value if they are present + if string and string[0] == "\"": + string = string[1:-1] + + return string + +def map_checksum_data(content_string): + arr = [] + + content_string = content_string.strip().strip('()').strip() + + tokens = re.split(r'\s+(?=(?:[^"]*"[^"]*")*[^"]*$)', content_string) + + current_rom = {} + i = 0 + while i < len(tokens): + if tokens[i] == 'name': + current_rom['name'] = tokens[i + 1].strip('"') + i += 2 + elif tokens[i] == 'size': + current_rom['size'] = int(tokens[i + 1]) + i += 2 + else: + checksum_key = tokens[i] + checksum_value = tokens[i + 1] if len(tokens) >= 6 else "0" + current_rom[checksum_key] = checksum_value + i += 2 + + arr.append(current_rom) + + return arr + +def map_key_values(content_string, arr): + # Split by newline into different pairs + temp = content_string.splitlines() + + # Add pairs to the dictionary if they are not parentheses + for pair in temp: + pair = pair.strip() + if pair == "(" or pair == ")": + continue + pair = list(map(str.strip, pair.split(None, 1))) + pair[1] = remove_quotes(pair[1]) + + # Handle duplicate keys (if the key is rom) and add values to a array instead + if pair[0] == "rom": + if 'rom' not in arr: + arr['rom'] = [] + arr['rom'].extend(map_checksum_data(pair[1])) + else: + arr[pair[0]] = pair[1].replace("\\", "") + + return arr + +def match_outermost_brackets(input): + """ + Parse DAT file and separate the contents each segment into an array + Segments are of the form `scummvm ( )`, `game ( )` etc. + """ + matches = [] + depth = 0 + inside_quotes = False + cur_index = 0 + + for i in range(len(input)): + char = input[i] + + if char == '(' and not inside_quotes: + if depth == 0: + cur_index = i + depth += 1 + elif char == ')' and not inside_quotes: + depth -= 1 + if depth == 0: + match = input[cur_index:i+1] + matches.append((match, cur_index)) + elif char == '"' and input[i - 1] != '\\': + inside_quotes = not inside_quotes + + return matches + +def parse_dat(dat_filepath): + """ + Take DAT filepath as input and return parsed data in the form of + associated arrays + """ + if not os.path.isfile(dat_filepath): + print("File not readable") + return + + with open(dat_filepath, "r", encoding="utf-8") as dat_file: + content = dat_file.read() + + header = {} + game_data = [] + resources = {} + + matches = match_outermost_brackets(content) + if matches: + for data_segment in matches: + if "clrmamepro" in content[data_segment[1] - 11: data_segment[1]] or \ + "scummvm" in content[data_segment[1] - 8: data_segment[1]]: + header = map_key_values(data_segment[0], header) + elif "game" in content[data_segment[1] - 5: data_segment[1]]: + temp = {} + temp = map_key_values(data_segment[0], temp) + game_data.append(temp) + elif "resource" in content[data_segment[1] - 9: data_segment[1]]: + temp = {} + temp = map_key_values(data_segment[0], temp) + resources[temp["name"]] = temp + # print(header, game_data, resources) + return header, game_data, resources, dat_filepath + +def main(): + parser = argparse.ArgumentParser(description="Process DAT files and interact with the database.") + parser.add_argument('--upload', nargs='+', help='Upload DAT file(s) to the database') + parser.add_argument('--match', nargs='+', help='Populate matching games in the database') + parser.add_argument('--user', help='Username for database') + parser.add_argument('-r', help="Recurse through directories", action='store_true') + parser.add_argument('--skiplog', help="Skip logging dups", action='store_true') + + args = parser.parse_args() + + if args.upload: + for filepath in args.upload: + db_insert(parse_dat(filepath), args.user, args.skiplog) + + if args.match: + for filepath in args.match: + match_fileset(parse_dat(filepath), args.user) + +if __name__ == "__main__": + main() diff --git a/db_functions.py b/db_functions.py new file mode 100644 index 00000000..99245df2 --- /dev/null +++ b/db_functions.py @@ -0,0 +1,993 @@ +import pymysql +import json +from collections import Counter +import getpass +import time +import hashlib +import os +from pymysql.converters import escape_string +from collections import defaultdict +import re + +def db_connect(): + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + conn = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + autocommit=False + ) + + return conn + +def get_checksum_props(checkcode, checksum): + checksize = 0 + checktype = checkcode + + if '-' in checkcode: + exploded_checkcode = checkcode.split('-') + last = exploded_checkcode.pop() + if last == '1M' or last.isdigit(): + checksize = last + + checktype = '-'.join(exploded_checkcode) + + # Detection entries have checktypes as part of the checksum prefix + if ':' in checksum: + prefix = checksum.split(':')[0] + checktype += "-" + prefix + + checksum = checksum.split(':')[1] + + return checksize, checktype, checksum + +def insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn): + # Set @engine_last if engine already present in table + exists = False + with conn.cursor() as cursor: + cursor.execute(f"SELECT id FROM engine WHERE engineid = '{engineid}'") + res = cursor.fetchone() + if res is not None: + exists = True + cursor.execute(f"SET @engine_last = '{res['id']}'") + + # Insert into table if not present + if not exists: + with conn.cursor() as cursor: + cursor.execute(f"INSERT INTO engine (name, engineid) VALUES ('{escape_string(engine_name)}', '{engineid}')") + cursor.execute("SET @engine_last = LAST_INSERT_ID()") + + # Insert into game + with conn.cursor() as cursor: + cursor.execute(f"INSERT INTO game (name, engine, gameid, extra, platform, language) VALUES ('{escape_string(title)}', @engine_last, '{gameid}', '{escape_string(extra)}', '{platform}', '{lang}')") + cursor.execute("SET @game_last = LAST_INSERT_ID()") + +def insert_fileset(src, detection, key, megakey, transaction, log_text, conn, ip='', username=None, skiplog=None): + status = "detection" if detection else src + game = "NULL" + key = "NULL" if key == "" else f"'{key}'" + megakey = "NULL" if megakey == "" else f"'{megakey}'" + + if detection: + status = "detection" + game = "@game_last" + + if status == "user": + game = "@game_last" + + # Check if key/megakey already exists, if so, skip insertion (no quotes on purpose) + if detection: + with conn.cursor() as cursor: + cursor.execute(f"SELECT id FROM fileset WHERE megakey = {megakey}") + + existing_entry = cursor.fetchone() + else: + with conn.cursor() as cursor: + cursor.execute(f"SELECT id FROM fileset WHERE `key` = {key}") + + existing_entry = cursor.fetchone() + + if existing_entry is not None: + existing_entry = existing_entry['id'] + with conn.cursor() as cursor: + cursor.execute(f"SET @fileset_last = {existing_entry}") + cursor.execute(f"DELETE FROM file WHERE fileset = {existing_entry}") + cursor.execute(f"UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last) WHERE id = {existing_entry}") + cursor.execute(f"UPDATE fileset SET status = 'detection' WHERE id = {existing_entry} AND status = 'obsolete'") + cursor.execute(f"SELECT status FROM fileset WHERE id = {existing_entry}") + status = cursor.fetchone()['status'] + if status == 'user': + add_usercount(existing_entry, conn) + category_text = f"Updated Fileset:{existing_entry}" + log_text = f"Updated Fileset:{existing_entry}, {log_text}" + user = f'cli:{getpass.getuser()}' if username is None else username + if not skiplog: + log_last = create_log(escape_string(category_text), user, escape_string(log_text), conn) + update_history(existing_entry, existing_entry, conn, log_last) + + return True + + # $game and $key should not be parsed as a mysql string, hence no quotes + query = f"INSERT INTO fileset (game, status, src, `key`, megakey, `timestamp`) VALUES ({game}, '{status}', '{src}', {key}, {megakey}, FROM_UNIXTIME(@fileset_time_last))" + with conn.cursor() as cursor: + cursor.execute(query) + cursor.execute("SET @fileset_last = LAST_INSERT_ID()") + + category_text = f"Uploaded from {src}" + with conn.cursor() as cursor: + cursor.execute("SELECT @fileset_last") + fileset_last = cursor.fetchone()['@fileset_last'] + + log_text = f"Created Fileset:{fileset_last}, {log_text}" + if src == 'user': + log_text = f"Created Fileset:{fileset_last}, from user: IP {ip}, {log_text}" + + user = f'cli:{getpass.getuser()}' if username is None else username + if not skiplog: + log_last = create_log(escape_string(category_text), user, escape_string(log_text), conn) + update_history(fileset_last, fileset_last, conn, log_last) + else: + update_history(0, fileset_last, conn) + with conn.cursor() as cursor: + cursor.execute(f"INSERT INTO transactions (`transaction`, fileset) VALUES ({transaction}, {fileset_last})") + + return True + +def insert_file(file, detection, src, conn): + # Find full md5, or else use first checksum value + checksum = "" + checksize = 5000 + checktype = "None" + if "md5" in file: + checksum = file["md5"] + else: + for key, value in file.items(): + if "md5" in key: + checksize, checktype, checksum = get_checksum_props(key, value) + break + + if not detection: + checktype = "None" + detection = 0 + detection_type = f"{checktype}-{checksize}" if checktype != "None" else f"{checktype}" + if punycode_need_encode(file['name']): + print(encode_punycode(file['name'])) + query = f"INSERT INTO file (name, size, checksum, fileset, detection, detection_type, `timestamp`) VALUES ('{encode_punycode(file['name'])}', '{file['size']}', '{checksum}', @fileset_last, {detection}, '{detection_type}', NOW())" + else: + query = f"INSERT INTO file (name, size, checksum, fileset, detection, detection_type, `timestamp`) VALUES ('{escape_string(file['name'])}', '{file['size']}', '{checksum}', @fileset_last, {detection}, '{detection_type}', NOW())" + with conn.cursor() as cursor: + cursor.execute(query) + + if detection: + with conn.cursor() as cursor: + cursor.execute(f"UPDATE fileset SET detection_size = {checksize} WHERE id = @fileset_last AND detection_size IS NULL") + with conn.cursor() as cursor: + cursor.execute("SET @file_last = LAST_INSERT_ID()") + +def insert_filechecksum(file, checktype, conn): + if checktype not in file: + return + + checksum = file[checktype] + checksize, checktype, checksum = get_checksum_props(checktype, checksum) + + query = f"INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (@file_last, '{checksize}', '{checktype}', '{checksum}')" + with conn.cursor() as cursor: + cursor.execute(query) + +def delete_filesets(conn): + query = "DELETE FROM fileset WHERE `delete` = TRUE" + with conn.cursor() as cursor: + cursor.execute(query) + +def my_escape_string(s: str) -> str: + """ + Escape strings + + Escape the following: + - escape char: \x81 + - unallowed filename chars: https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words + - control chars < 0x20 + """ + new_name = "" + for char in s: + if char == "\x81": + new_name += "\x81\x79" + elif char in '/":*|\\?%<>\x7f' or ord(char) < 0x20 or (ord(char) & 0x80): + new_name += "\x81" + chr(0x80 + ord(char)) + else: + new_name += char + return new_name + + +def encode_punycode(orig): + """ + Punyencode strings + + - escape special characters and + - ensure filenames can't end in a space or dot + """ + s = my_escape_string(orig) + encoded = s.encode("punycode").decode("ascii") + # punyencoding adds an '-' at the end when there are no special chars + # don't use it for comparing + compare = encoded + if encoded.endswith("-"): + compare = encoded[:-1] + if orig != compare or compare[-1] in " .": + return "xn--" + encoded + return orig + +def punycode_need_encode(orig): + """ + A filename needs to be punyencoded when it: + + - contains a char that should be escaped or + - ends with a dot or a space. + """ + if orig != escape_string(orig): + return True + if orig[-1] in " .": + return True + return False + +def create_log(category, user, text, conn): + query = f"INSERT INTO log (`timestamp`, category, user, `text`) VALUES (FROM_UNIXTIME({int(time.time())}), '{escape_string(category)}', '{escape_string(user)}', '{escape_string(text)}')" + with conn.cursor() as cursor: + try: + cursor.execute(query) + conn.commit() + except Exception as e: + conn.rollback() + print(f"Creating log failed: {e}") + log_last = None + else: + cursor.execute("SELECT LAST_INSERT_ID()") + log_last = cursor.fetchone()['LAST_INSERT_ID()'] + return log_last + +def update_history(source_id, target_id, conn, log_last=None): + query = f"INSERT INTO history (`timestamp`, fileset, oldfileset, log) VALUES (NOW(), {target_id}, {source_id}, {log_last if log_last is not None else 0})" + with conn.cursor() as cursor: + try: + cursor.execute(query) + conn.commit() + except Exception as e: + conn.rollback() + print(f"Creating log failed: {e}") + log_last = None + else: + cursor.execute("SELECT LAST_INSERT_ID()") + log_last = cursor.fetchone()['LAST_INSERT_ID()'] + return log_last + +def get_all_related_filesets(fileset_id, conn, visited=None): + if visited is None: + visited = set() + + if fileset_id in visited or fileset_id == 0: + return [] + + visited.add(fileset_id) + + related_filesets = [fileset_id] + try: + with conn.cursor() as cursor: + cursor.execute(f"SELECT fileset, oldfileset FROM history WHERE fileset = {fileset_id} OR oldfileset = {fileset_id}") + history_records = cursor.fetchall() + + for record in history_records: + if record['fileset'] not in visited: + related_filesets.extend(get_all_related_filesets(record['fileset'], conn, visited)) + if record['oldfileset'] not in visited: + related_filesets.extend(get_all_related_filesets(record['oldfileset'], conn, visited)) + except pymysql.err.InterfaceError: + print("Connection lost, reconnecting...") + try: + conn = db_connect() # Reconnect if the connection is lost + except Exception as e: + print(f"Failed to reconnect: {e}") + + except Exception as e: + print(f"Error fetching related filesets: {e}") + + return related_filesets + +def convert_log_text_to_links(log_text): + log_text = re.sub(r'Fileset:(\d+)', r'Fileset:\1', log_text) + log_text = re.sub(r'user:(\w+)', r'user:\1', log_text) + log_text = re.sub(r'Transaction:(\d+)', r'Transaction:\1', log_text) + return log_text + +def calc_key(fileset): + key_string = "" + + for key, value in fileset.items(): + if key in ['engineid', 'gameid', 'rom']: + continue + key_string += ':' + str(value) + + files = fileset['rom'] + for file in files: + for key, value in file.items(): + key_string += ':' + str(value) + + key_string = key_string.strip(':') + return hashlib.md5(key_string.encode()).hexdigest() + +def calc_megakey(fileset): + key_string = f":{fileset['platform']}:{fileset['language']}" + if 'rom' in fileset.keys(): + for file in fileset['rom']: + for key, value in file.items(): + key_string += ':' + str(value) + elif 'files' in fileset.keys(): + for file in fileset['files']: + for key, value in file.items(): + key_string += ':' + str(value) + + key_string = key_string.strip(':') + return hashlib.md5(key_string.encode()).hexdigest() + +def db_insert(data_arr, username=None, skiplog=False): + header = data_arr[0] + game_data = data_arr[1] + resources = data_arr[2] + filepath = data_arr[3] + + try: + conn = db_connect() + except Exception as e: + print(f"Failed to connect to database: {e}") + return + + try: + author = header["author"] + version = header["version"] + except KeyError as e: + print(f"Missing key in header: {e}") + return + + src = "dat" if author not in ["scan", "scummvm"] else author + + detection = (src == "scummvm") + status = "detection" if detection else src + + conn.cursor().execute(f"SET @fileset_time_last = {int(time.time())}") + + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + temp = cursor.fetchone()['MAX(`transaction`)'] + if temp == None: + temp = 0 + transaction_id = temp + 1 + + category_text = f"Uploaded from {src}" + log_text = f"Started loading DAT file, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Transaction: {transaction_id}" + + user = f'cli:{getpass.getuser()}' if username is None else username + create_log(escape_string(category_text), user, escape_string(log_text), conn) + + for fileset in game_data: + if detection: + engine_name = fileset["engine"] + engineid = fileset["sourcefile"] + gameid = fileset["name"] + title = fileset["title"] + extra = fileset["extra"] + platform = fileset["platform"] + lang = fileset["language"] + + insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn) + elif src == "dat": + if 'romof' in fileset and fileset['romof'] in resources: + fileset["rom"] = fileset["rom"] + resources[fileset["romof"]]["rom"] + + key = calc_key(fileset) if not detection else "" + megakey = calc_megakey(fileset) if detection else "" + log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}." + + if insert_fileset(src, detection, key, megakey, transaction_id, log_text, conn, username=username, skiplog=skiplog): + for file in fileset["rom"]: + insert_file(file, detection, src, conn) + for key, value in file.items(): + if key not in ["name", "size"]: + insert_filechecksum(file, key, conn) + + if detection: + conn.cursor().execute("UPDATE fileset SET status = 'obsolete' WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last) AND status = 'detection'") + cur = conn.cursor() + + try: + cur.execute(f"SELECT COUNT(fileset) from transactions WHERE `transaction` = {transaction_id}") + fileset_insertion_count = cur.fetchone()['COUNT(fileset)'] + category_text = f"Uploaded from {src}" + log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" + except Exception as e: + print("Inserting failed:", e) + else: + user = f'cli:{getpass.getuser()}' if username is None else username + create_log(escape_string(category_text), user, escape_string(log_text), conn) + +def compare_filesets(id1, id2, conn): + with conn.cursor() as cursor: + cursor.execute(f"SELECT name, size, checksum FROM file WHERE fileset = '{id1}'") + fileset1 = cursor.fetchall() + cursor.execute(f"SELECT name, size, checksum FROM file WHERE fileset = '{id2}'") + fileset2 = cursor.fetchall() + + # Sort filesets on checksum + fileset1.sort(key=lambda x: x[2]) + fileset2.sort(key=lambda x: x[2]) + + if len(fileset1) != len(fileset2): + return False + + for i in range(len(fileset1)): + # If checksums do not match + if fileset1[i][2] != fileset2[i][2]: + return False + + return True + +def status_to_match(status): + order = ["detection", "dat", "scan", "partialmatch", "fullmatch", "user"] + return order[:order.index(status)] + +def find_matching_game(game_files): + matching_games = [] # All matching games + matching_filesets = [] # All filesets containing one file from game_files + matches_count = 0 # Number of files with a matching detection entry + + conn = db_connect() + + for file in game_files: + checksum = file[1] + + query = f"SELECT file.fileset as file_fileset FROM filechecksum JOIN file ON filechecksum.file = file.id WHERE filechecksum.checksum = '{checksum}' AND file.detection = TRUE" + with conn.cursor() as cursor: + cursor.execute(query) + records = cursor.fetchall() + + # If file is not part of detection entries, skip it + if len(records) == 0: + continue + + matches_count += 1 + for record in records: + matching_filesets.append(record[0]) + + # Check if there is a fileset_id that is present in all results + for key, value in Counter(matching_filesets).items(): + with conn.cursor() as cursor: + cursor.execute(f"SELECT COUNT(file.id) FROM file JOIN fileset ON file.fileset = fileset.id WHERE fileset.id = '{key}'") + count_files_in_fileset = cursor.fetchone()['COUNT(file.id)'] + + # We use < instead of != since one file may have more than one entry in the fileset + # We see this in Drascula English version, where one entry is duplicated + if value < matches_count or value < count_files_in_fileset: + continue + + with conn.cursor() as cursor: + cursor.execute(f"SELECT engineid, game.id, gameid, platform, language, `key`, src, fileset.id as fileset FROM game JOIN fileset ON fileset.game = game.id JOIN engine ON engine.id = game.engine WHERE fileset.id = '{key}'") + records = cursor.fetchall() + + matching_games.append(records[0]) + + if len(matching_games) != 1: + return matching_games + + # Check the current fileset priority with that of the match + with conn.cursor() as cursor: + cursor.execute(f"SELECT id FROM fileset, ({query}) AS res WHERE id = file_fileset AND status IN ({', '.join(['%s']*len(game_files[3]))})", status_to_match(game_files[3])) + records = cursor.fetchall() + + # If priority order is correct + if len(records) != 0: + return matching_games + + if compare_filesets(matching_games[0]['fileset'], game_files[0][0], conn): + with conn.cursor() as cursor: + cursor.execute(f"UPDATE fileset SET `delete` = TRUE WHERE id = {game_files[0][0]}") + return [] + + return matching_games + +def merge_filesets(detection_id, dat_id): + conn = db_connect() + + try: + with conn.cursor() as cursor: + cursor.execute(f"SELECT DISTINCT(filechecksum.checksum), checksize, checktype FROM filechecksum JOIN file on file.id = filechecksum.file WHERE fileset = '{detection_id}'") + detection_files = cursor.fetchall() + + for file in detection_files: + checksum = file[0] + checksize = file[1] + checktype = file[2] + + cursor.execute(f"DELETE FROM file WHERE checksum = '{checksum}' AND fileset = {detection_id} LIMIT 1") + cursor.execute(f"UPDATE file JOIN filechecksum ON filechecksum.file = file.id SET detection = TRUE, checksize = {checksize}, checktype = '{checktype}' WHERE fileset = '{dat_id}' AND filechecksum.checksum = '{checksum}'") + + cursor.execute(f"INSERT INTO history (`timestamp`, fileset, oldfileset) VALUES (FROM_UNIXTIME({int(time.time())}), {dat_id}, {detection_id})") + cursor.execute("SELECT LAST_INSERT_ID()") + history_last = cursor.fetchone()['LAST_INSERT_ID()'] + + cursor.execute(f"UPDATE history SET fileset = {dat_id} WHERE fileset = {detection_id}") + cursor.execute(f"DELETE FROM fileset WHERE id = {detection_id}") + + conn.commit() + except Exception as e: + conn.rollback() + print(f"Error merging filesets: {e}") + finally: + # conn.close() + pass + + return history_last + + +def populate_matching_games(): + conn = db_connect() + + # Getting unmatched filesets + unmatched_filesets = [] + + with conn.cursor() as cursor: + cursor.execute("SELECT fileset.id, filechecksum.checksum, src, status FROM fileset JOIN file ON file.fileset = fileset.id JOIN filechecksum ON file.id = filechecksum.file WHERE fileset.game IS NULL AND status != 'user'") + unmatched_files = cursor.fetchall() + + # Splitting them into different filesets + i = 0 + while i < len(unmatched_files): + cur_fileset = unmatched_files[i][0] + temp = [] + while i < len(unmatched_files) and cur_fileset == unmatched_files[i][0]: + temp.append(unmatched_files[i]) + i += 1 + unmatched_filesets.append(temp) + + for fileset in unmatched_filesets: + matching_games = find_matching_game(fileset) + + if len(matching_games) != 1: # If there is no match/non-unique match + continue + + matched_game = matching_games[0] + + # Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch) + status = fileset[0][2] + if fileset[0][2] == "dat": + status = "partialmatch" + elif fileset[0][2] == "scan": + status = "fullmatch" + + # Convert NULL values to string with value NULL for printing + matched_game = {k: 'NULL' if v is None else v for k, v in matched_game.items()} + + category_text = f"Matched from {fileset[0][2]}" + log_text = f"Matched game {matched_game['engineid']}:\n{matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']}\nvariant {matched_game['key']}. State {status}. Fileset:{fileset[0][0]}." + + # Updating the fileset.game value to be $matched_game["id"] + query = f"UPDATE fileset SET game = {matched_game['id']}, status = '{status}', `key` = '{matched_game['key']}' WHERE id = {fileset[0][0]}" + + history_last = merge_filesets(matched_game["fileset"], fileset[0][0]) + + if cursor.execute(query): + user = f'cli:{getpass.getuser()}' + + create_log("Fileset merge", user, escape_string(f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}"), conn) + + # Matching log + log_last = create_log(escape_string(conn, category_text), user, escape_string(conn, log_text)) + + # Add log id to the history table + cursor.execute(f"UPDATE history SET log = {log_last} WHERE id = {history_last}") + + try: + conn.commit() + except: + print("Updating matched games failed") + +def match_fileset(data_arr, username=None): + header, game_data, resources, filepath = data_arr + + try: + conn = db_connect() + except Exception as e: + print(f"Failed to connect to database: {e}") + return + + try: + author = header["author"] + version = header["version"] + except KeyError as e: + print(f"Missing key in header: {e}") + return + + src = "dat" if author not in ["scan", "scummvm"] else author + detection = (src == "scummvm") + source_status = "detection" if detection else src + + conn.cursor().execute(f"SET @fileset_time_last = {int(time.time())}") + + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + transaction_id = cursor.fetchone()['MAX(`transaction`)'] + transaction_id = transaction_id + 1 if transaction_id else 1 + + category_text = f"Uploaded from {src}" + log_text = f"Started loading DAT file, size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}. Transaction: {transaction_id}" + + user = f'cli:{getpass.getuser()}' if username is None else username + create_log(escape_string(category_text), user, escape_string(log_text), conn) + + for fileset in game_data: + process_fileset(fileset, resources, detection, src, conn, transaction_id, filepath, author, version, source_status, user) + finalize_fileset_insertion(conn, transaction_id, src, filepath, author, version, source_status, user) + +def process_fileset(fileset, resources, detection, src, conn, transaction_id, filepath, author, version, source_status, user): + if detection: + insert_game_data(fileset, conn) + elif src == "dat" and 'romof' in fileset and fileset['romof'] in resources: + fileset["rom"] += resources[fileset["romof"]]["rom"] + + key = calc_key(fileset) if not detection else "" + megakey = calc_megakey(fileset) if detection else "" + log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}." + if src != "dat": + matched_map = find_matching_filesets(fileset, conn, src) + else: + matched_map = matching_set(fileset, conn) + + + insert_new_fileset(fileset, conn, detection, src, key, megakey, transaction_id, log_text, user) + with conn.cursor() as cursor: + cursor.execute("SET @fileset_last = LAST_INSERT_ID()") + cursor.execute("SELECT LAST_INSERT_ID()") + fileset_last = cursor.fetchone()['LAST_INSERT_ID()'] + if matched_map: + handle_matched_filesets(fileset_last, matched_map, fileset, conn, detection, src, key, megakey, transaction_id, log_text, user) + +def insert_game_data(fileset, conn): + engine_name = fileset["engine"] + engineid = fileset["sourcefile"] + gameid = fileset["name"] + title = fileset["title"] + extra = fileset["extra"] + platform = fileset["platform"] + lang = fileset["language"] + insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn) + +def find_matching_filesets(fileset, conn, status): + matched_map = defaultdict(list) + if status != "user": + state = """'detection', 'dat', 'scan', 'partial', 'full', 'obsolete'""" + else: + state = """'partial', 'full', 'dat'""" + with conn.cursor() as cursor: + for file in fileset["rom"]: + matched_set = set() + for key, value in file.items(): + if key not in ["name", "size", "sha1", "crc"]: + checksum = file[key] + checktype = key + checksize, checktype, checksum = get_checksum_props(checktype, checksum) + query = f"""SELECT DISTINCT fs.id AS fileset_id + FROM fileset fs + JOIN file f ON fs.id = f.fileset + JOIN filechecksum fc ON f.id = fc.file + WHERE fc.checksum = '{checksum}' AND fc.checktype = '{checktype}' + AND fs.status IN ({state})""" + cursor.execute(query) + records = cursor.fetchall() + if records: + for record in records: + matched_set.add(record['fileset_id']) + + for id in matched_set: + matched_map[id].append(file) + + return matched_map + +def matching_set(fileset, conn): + matched_map = defaultdict(list) + with conn.cursor() as cursor: + for file in fileset["rom"]: + matched_set = set() + if "md5" in file: + checksum = file["md5"] + size = file["size"] + query = f""" + SELECT DISTINCT fs.id AS fileset_id + FROM fileset fs + JOIN file f ON fs.id = f.fileset + JOIN filechecksum fc ON f.id = fc.file + WHERE fc.checksum = '{checksum}' AND fc.checktype = 'md5' + AND fc.checksize > {size} + AND fs.status = 'detection' + """ + cursor.execute(query) + records = cursor.fetchall() + if records: + for record in records: + matched_set.add(record['fileset_id']) + for id in matched_set: + matched_map[id].append(file) + return matched_map + +def handle_matched_filesets(fileset_last, matched_map, fileset, conn, detection, src, key, megakey, transaction_id, log_text, user): + matched_list = sorted(matched_map.items(), key=lambda x: len(x[1]), reverse=True) + is_full_matched = False + with conn.cursor() as cursor: + for matched_fileset_id, matched_count in matched_list: + if is_full_matched: + break + cursor.execute(f"SELECT status FROM fileset WHERE id = {matched_fileset_id}") + status = cursor.fetchone()['status'] + cursor.execute(f"SELECT COUNT(file.id) FROM file WHERE fileset = {matched_fileset_id}") + count = cursor.fetchone()['COUNT(file.id)'] + + if status in ['detection', 'obsolete'] and count == len(matched_count): + is_full_matched = True + update_fileset_status(cursor, matched_fileset_id, 'full' if src != "dat" else "partial") + populate_file(fileset, matched_fileset_id, conn, detection) + log_matched_fileset(src, fileset_last, matched_fileset_id, 'full' if src != "dat" else "partial", user, conn) + delete_original_fileset(fileset_last, conn) + elif status == 'full' and len(fileset['rom']) == count: + is_full_matched = True + log_matched_fileset(src, fileset_last, matched_fileset_id, 'full', user, conn) + delete_original_fileset(fileset_last, conn) + return + elif (status == 'partial') and count == len(matched_count): + is_full_matched = True + update_fileset_status(cursor, matched_fileset_id, 'full') + populate_file(fileset, matched_fileset_id, conn, detection) + log_matched_fileset(src, fileset_last, matched_fileset_id, 'full', user, conn) + delete_original_fileset(fileset_last, conn) + elif status == 'scan' and count == len(matched_count): + log_matched_fileset(src, fileset_last, matched_fileset_id, 'full', user, conn) + elif src == 'dat': + log_matched_fileset(src, fileset_last, matched_fileset_id, 'partial matched', user, conn) + +def delete_original_fileset(fileset_id, conn): + with conn.cursor() as cursor: + cursor.execute(f"DELETE FROM file WHERE fileset = {fileset_id}") + cursor.execute(f"DELETE FROM fileset WHERE id = {fileset_id}") + +def update_fileset_status(cursor, fileset_id, status): + cursor.execute(f""" + UPDATE fileset SET + status = '{status}', + `timestamp` = FROM_UNIXTIME({int(time.time())}) + WHERE id = {fileset_id} + """) + +def populate_file(fileset, fileset_id, conn, detection): + with conn.cursor() as cursor: + cursor.execute(f"SELECT * FROM file WHERE fileset = {fileset_id}") + target_files = cursor.fetchall() + target_files_dict = {} + for target_file in target_files: + cursor.execute(f"SELECT * FROM filechecksum WHERE file = {target_file['id']}") + target_checksums = cursor.fetchall() + for checksum in target_checksums: + target_files_dict[checksum['checksum']] = target_file + target_files_dict[target_file['id']] = f"{checksum['checktype']}-{checksum['checksize']}" + for file in fileset['rom']: + file_exists = False + checksum = "" + checksize = 5000 + checktype = "None" + if "md5" in file: + checksum = file["md5"] + else: + for key, value in file.items(): + if "md5" in key: + checksize, checktype, checksum = get_checksum_props(key, value) + break + if not detection: + checktype = "None" + detection = 0 + detection_type = f"{checktype}-{checksize}" if checktype != "None" else f"{checktype}" + if punycode_need_encode(file['name']): + print(encode_punycode(file['name'])) + query = f"INSERT INTO file (name, size, checksum, fileset, detection, detection_type, `timestamp`) VALUES ('{encode_punycode(file['name'])}', '{file['size']}', '{checksum}', @fileset_last, {detection}, '{detection_type}', NOW())" + else: + query = f"INSERT INTO file (name, size, checksum, fileset, detection, detection_type, `timestamp`) VALUES ('{escape_string(file['name'])}', '{file['size']}', '{checksum}', @fileset_last, {detection}, '{detection_type}', NOW())" + cursor.execute(query) + cursor.execute("SET @file_last = LAST_INSERT_ID()") + cursor.execute("SELECT @file_last AS file_id") + file_id = cursor.fetchone()['file_id'] + target_id = None + for key, value in file.items(): + if key not in ["name", "size"]: + insert_filechecksum(file, key, conn) + if value in target_files_dict and not file_exists: + file_exists = True + target_id = target_files_dict[value]['id'] + cursor.execute(f"DELETE FROM file WHERE id = {target_files_dict[value]['id']}") + + if file_exists: + cursor.execute(f"UPDATE file SET detection = 1 WHERE id = {file_id}") + cursor.execute(f"UPDATE file SET detection_type = '{target_files_dict[target_id]}' WHERE id = {file_id}") + else: + cursor.execute(f"UPDATE file SET detection_type = 'None' WHERE id = {file_id}") + +def insert_new_fileset(fileset, conn, detection, src, key, megakey, transaction_id, log_text, user, ip=''): + if insert_fileset(src, detection, key, megakey, transaction_id, log_text, conn, username=user, ip=ip): + for file in fileset["rom"]: + insert_file(file, detection, src, conn) + for key, value in file.items(): + if key not in ["name", "size", "sha1", "crc"]: + insert_filechecksum(file, key, conn) + +def log_matched_fileset(src, fileset_last, fileset_id, state, user, conn): + category_text = f"Matched from {src}" + log_text = f"Matched Fileset:{fileset_id}. State {state}." + log_last = create_log(escape_string(category_text), user, escape_string(log_text), conn) + update_history(fileset_last, fileset_id, conn, log_last) + +def finalize_fileset_insertion(conn, transaction_id, src, filepath, author, version, source_status, user): + with conn.cursor() as cursor: + cursor.execute(f"SELECT COUNT(fileset) from transactions WHERE `transaction` = {transaction_id}") + fileset_insertion_count = cursor.fetchone()['COUNT(fileset)'] + category_text = f"Uploaded from {src}" + if src != 'user': + log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" + create_log(escape_string(category_text), user, escape_string(log_text), conn) + # conn.close() + +def user_integrity_check(data, ip, game_metadata=None): + src = "user" + source_status = src + new_files = [] + + for file in data["files"]: + new_file = { + "name": file["name"], + "size": file["size"] + } + for checksum in file["checksums"]: + checksum_type = checksum["type"] + checksum_value = checksum["checksum"] + new_file[checksum_type] = checksum_value + + new_files.append(new_file) + + data["rom"] = new_files + key = calc_key(data) + try: + conn = db_connect() + except Exception as e: + print(f"Failed to connect to database: {e}") + return + + conn.cursor().execute(f"SET @fileset_time_last = {int(time.time())}") + + try: + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + transaction_id = cursor.fetchone()['MAX(`transaction`)'] + 1 + + category_text = f"Uploaded from {src}" + log_text = f"Started loading file, State {source_status}. Transaction: {transaction_id}" + + user = f'cli:{getpass.getuser()}' + + create_log(escape_string(category_text), user, escape_string(log_text), conn) + + matched_map = find_matching_filesets(data, conn, src) + + # show matched, missing, extra + extra_map = defaultdict(list) + missing_map = defaultdict(list) + extra_set = set() + missing_set = set() + + for fileset_id in matched_map.keys(): + cursor.execute(f"SELECT * FROM file WHERE fileset = {fileset_id}") + target_files = cursor.fetchall() + target_files_dict = {} + for target_file in target_files: + cursor.execute(f"SELECT * FROM filechecksum WHERE file = {target_file['id']}") + target_checksums = cursor.fetchall() + for checksum in target_checksums: + target_files_dict[checksum['checksum']] = target_file + # target_files_dict[target_file['id']] = f"{checksum['checktype']}-{checksum['checksize']}" + + # Collect all the checksums from data['files'] + data_files_set = set() + for file in data["files"]: + for checksum_info in file["checksums"]: + checksum = checksum_info["checksum"] + checktype = checksum_info["type"] + checksize, checktype, checksum = get_checksum_props(checktype, checksum) + data_files_set.add(checksum) + + # Identify missing files + matched_names = set() + for checksum, target_file in target_files_dict.items(): + if checksum not in data_files_set: + if target_file['name'] not in matched_names: + missing_set.add(target_file['name']) + else: + missing_set.discard(target_file['name']) + else: + matched_names.add(target_file['name']) + + for tar in missing_set: + missing_map[fileset_id].append({'name': tar}) + + # Identify extra files + for file in data['files']: + file_exists = False + for checksum_info in file["checksums"]: + checksum = checksum_info["checksum"] + checktype = checksum_info["type"] + checksize, checktype, checksum = get_checksum_props(checktype, checksum) + if checksum in target_files_dict and not file_exists: + file_exists = True + if not file_exists: + extra_set.add(file['name']) + + for extra in extra_set: + extra_map[fileset_id].append({'name': extra}) + if game_metadata: + platform = game_metadata['platform'] + lang = game_metadata['language'] + gameid = game_metadata['gameid'] + engineid = game_metadata['engineid'] + extra_info = game_metadata['extra'] + engine_name = " " + title = " " + insert_game(engine_name, engineid, title, gameid, extra_info, platform, lang, conn) + + # handle different scenarios + if len(matched_map) == 0: + insert_new_fileset(data, conn, None, src, key, None, transaction_id, log_text, user, ip) + return matched_map, missing_map, extra_map + + matched_list = sorted(matched_map.items(), key=lambda x: len(x[1]), reverse=True) + most_matched = matched_list[0] + matched_fileset_id, matched_count = most_matched[0], most_matched[1] + cursor.execute(f"SELECT status FROM fileset WHERE id = {matched_fileset_id}") + status = cursor.fetchone()['status'] + + cursor.execute(f"SELECT COUNT(file.id) FROM file WHERE fileset = {matched_fileset_id}") + count = cursor.fetchone()['COUNT(file.id)'] + if status == "full" and count == matched_count: + log_matched_fileset(src, matched_fileset_id, matched_fileset_id, 'full', user, conn) + elif status == "partial" and count == matched_count: + populate_file(data, matched_fileset_id, conn, None, src) + log_matched_fileset(src, matched_fileset_id, matched_fileset_id, 'partial', user, conn) + elif status == "user" and count == matched_count: + add_usercount(matched_fileset_id, conn) + log_matched_fileset(src, matched_fileset_id, matched_fileset_id, 'user', user, conn) + else: + insert_new_fileset(data, conn, None, src, key, None, transaction_id, log_text, user, ip) + finalize_fileset_insertion(conn, transaction_id, src, None, user, 0, source_status, user) + except Exception as e: + conn.rollback() + print(f"Error processing user data: {e}") + finally: + category_text = f"Uploaded from {src}" + log_text = f"Completed loading file, State {source_status}. Transaction: {transaction_id}" + create_log(escape_string(category_text), user, escape_string(log_text), conn) + # conn.close() + return matched_map, missing_map, extra_map + +def add_usercount(fileset, conn): + with conn.cursor() as cursor: + cursor.execute(f"UPDATE fileset SET user_count = COALESCE(user_count, 0) + 1 WHERE id = {fileset}") + cursor.execute(f"SELECT user_count from fileset WHERE id = {fileset}") + count = cursor.fetchone()['user_count'] + if count >= 3: + cursor.execute(f"UPDATE fileset SET status = 'ReadyForReview' WHERE id = {fileset}") \ No newline at end of file diff --git a/endpoints/validate.php b/endpoints/validate.php deleted file mode 100644 index 1963a5ce..00000000 --- a/endpoints/validate.php +++ /dev/null @@ -1,166 +0,0 @@ - -1, - "success" => 0, - "empty" => 2, - "no_metadata" => 3, -); - -$json_string = file_get_contents('php://input'); -$json_object = json_decode($json_string); - -$ip = $_SERVER['REMOTE_ADDR']; -// Take only first 3 bytes, set 4th byte as '.X' -// FIXME: Assumes IPv4 -$ip = implode('.', array_slice(explode('.', $ip), 0, 3)) . '.X'; - -$game_metadata = array(); -foreach ($json_object as $key => $value) { - if ($key == 'files') - continue; - - $game_metadata[$key] = $value; -} - -$json_response = array( - 'error' => $error_codes['success'], - 'files' => array() -); - -if (count($game_metadata) == 0) { - if (count($json_object->files) == 0) { - $json_response['error'] = $error_codes['empty']; - unset($json_response['files']); - $json_response['status'] = 'empty_fileset'; - - - $json_response = json_encode($json_response); - echo $json_response; - return; - } - - $json_response['error'] = $error_codes['no_metadata']; - unset($json_response['files']); - $json_response['status'] = 'no_metadata'; - - $fileset_id = user_insert_fileset($json_object->files, $ip, $conn); - $json_response['fileset'] = $fileset_id; - - $json_response = json_encode($json_response); - echo $json_response; - return; -} - -// Find game(s) that fit the metadata -$query = "SELECT game.id FROM game -JOIN engine ON game.engine = engine.id -WHERE gameid = '{$game_metadata['gameid']}' -AND engineid = '{$game_metadata['engineid']}' -AND platform = '{$game_metadata['platform']}' -AND language = '{$game_metadata['language']}'"; -$games = $conn->query($query); - -if ($games->num_rows == 0) { - $json_response['error'] = $error_codes['unknown']; - unset($json_response['files']); - $json_response['status'] = 'unknown_variant'; - - $fileset_id = user_insert_fileset($json_object->files, $ip, $conn); - $json_response['fileset'] = $fileset_id; -} - -// Check if all files in the (first) fileset are present with user -while ($game = $games->fetch_array()) { - $fileset = $conn->query("SELECT file.id, name, size FROM file - JOIN fileset ON fileset.id = file.fileset - WHERE fileset.game = {$game['id']} AND - (status = 'fullmatch' OR status = 'partialmatch' OR status = 'detection')"); - - if ($fileset->num_rows == 0) - continue; - - // Convert checktype, checksize to checkcode - $fileset = $fileset->fetch_all(MYSQLI_ASSOC); - foreach (array_values($fileset) as $index => $file) { - $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype - FROM filechecksum WHERE file = {$file['id']}"); - - while ($spec_checksum = $spec_checksum_res->fetch_assoc()) { - $fileset[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum']; - } - } - - $file_object = $json_object->files; - - // Sort the filesets by filename - usort($file_object, function ($a, $b) { - return strcmp($a->name, $b->name); - }); - usort($fileset, function ($a, $b) { - return strcmp($a['name'], $b['name']); - }); - - for ($i = 0, $j = 0; $i < count($fileset) && $j < count($file_object); $i++, $j++) { - $status = 'ok'; - $db_file = $fileset[$i]; - $user_file = $file_object[$j]; - $filename = strtolower($user_file->name); - - if (strtolower($db_file['name']) != $filename) { - if (strtolower($db_file['name']) > $filename) { - $status = 'unknown_file'; - $i--; // Retain same db_file for next iteration - } - else { - $status = 'missing'; - $filename = $db_file['name']; - $j--; // Retain same user_file for next iteration - } - } - elseif ($db_file['size'] != $user_file->size && $status == 'ok') { - $status = 'size_mismatch'; - } - - if ($status == 'ok') { - foreach ($user_file->checksums as $checksum_data) { - foreach ($checksum_data as $key => $value) { - $user_checkcode = $checksum_data->type; - // If it's not the full checksum - if (strpos($user_checkcode, '-') !== false) - continue; - - $user_checksum = $checksum_data->checksum; - $user_checkcode .= '-0'; - - if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0) - $status = 'checksum_mismatch'; - - break; - } - } - } - - if ($status != 'ok') { - $json_response['error'] = 1; - - $fileset_id = user_insert_fileset($json_object->files, $ip, $conn); - $json_response['fileset'] = $fileset_id; - } - - array_push($json_response['files'], array('status' => $status, 'name' => $filename)); - } - - break; -} - -$json_response = json_encode($json_response); -echo $json_response; -?> - diff --git a/fileset.php b/fileset.php deleted file mode 100644 index 73a09e52..00000000 --- a/fileset.php +++ /dev/null @@ -1,215 +0,0 @@ -\n"; -echo "\n"; -echo "\n"; - - -$mysql_cred = json_decode(file_get_contents(__DIR__ . '/mysql_config.json'), true); -$servername = $mysql_cred["servername"]; -$username = $mysql_cred["username"]; -$password = $mysql_cred["password"]; -$dbname = $mysql_cred["dbname"]; - -// Create connection -mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); -$conn = new mysqli($servername, $username, $password); -$conn->set_charset('utf8mb4'); -$conn->autocommit(FALSE); - -// Check connection -if ($conn->connect_errno) { - die("Connect failed: " . $conn->connect_error); -} - -$conn->query("USE " . $dbname); - -$min_id = $conn->query("SELECT MIN(id) FROM fileset")->fetch_array()[0]; -if (!isset($_GET['id'])) { - $id = $min_id; -} -else { - $max_id = $conn->query("SELECT MAX(id) FROM fileset")->fetch_array()[0]; - $id = max($min_id, min($_GET['id'], $max_id)); - if ($conn->query("SELECT id FROM fileset WHERE id = {$id}")->num_rows == 0) - $id = $conn->query("SELECT fileset FROM history WHERE oldfileset = {$id}")->fetch_array()[0]; -} - -$history = $conn->query("SELECT `timestamp`, oldfileset, log -FROM history WHERE fileset = {$id} -ORDER BY `timestamp`"); - - -// Display fileset details -echo "

Fileset: {$id}

"; - -$result = $conn->query("SELECT * FROM fileset WHERE id = {$id}")->fetch_assoc(); - -echo "

Fileset details

"; -echo "\n"; -if ($result['game']) { - $temp = $conn->query("SELECT game.name as 'game name', engineid, gameid, extra, platform, language -FROM fileset JOIN game ON game.id = fileset.game JOIN engine ON engine.id = game.engine -WHERE fileset.id = {$id}"); - $result = array_merge($result, $temp->fetch_assoc()); -} -else { - unset($result['key']); - unset($result['status']); - unset($result['delete']); -} - -foreach (array_keys($result) as $column) { - if ($column == 'id' || $column == 'game') - continue; - - echo "\n"; -} - -echo "\n"; -foreach ($result as $column => $value) { - if ($column == 'id' || $column == 'game') - continue; - - echo ""; -} -echo "\n"; -echo "
{$column}
{$value}
\n"; - -echo "

Files in the fileset

"; -echo "
"; -// Preserve GET variables on form submit -foreach ($_GET as $k => $v) { - if ($k == 'widetable') - continue; - - $k = htmlspecialchars($k); - $v = htmlspecialchars($v); - echo ""; -} - -// Come up with a better solution to set widetable=true on button click -// Currently uses hidden text input -if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') { - echo ""; - echo ""; -} -else { - echo ""; - echo ""; -} - -echo "
"; - -// Table -echo "\n"; - -$result = $conn->query("SELECT file.id, name, size, checksum, detection - FROM file WHERE fileset = {$id}")->fetch_all(MYSQLI_ASSOC); - -if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') { - foreach (array_values($result) as $index => $file) { - $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype - FROM filechecksum WHERE file = {$file['id']}"); - - while ($spec_checksum = $spec_checksum_res->fetch_assoc()) { - // md5-0 is skipped since it is already shown as file.checksum - if ($spec_checksum['checksize'] == 0) - continue; - - $result[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum']; - } - } -} - -$counter = 1; -foreach ($result as $row) { - if ($counter == 1) { - echo "\n"; - } - } - - echo "\n"; - echo "\n"; - foreach ($row as $key => $value) { - if ($key == 'id') - continue; - - echo "\n"; - } - echo "\n"; - - $counter++; -} -echo "
\n"; // Numbering column - foreach (array_keys($row) as $index => $key) { - if ($key == 'id') - continue; - - echo "{$key}
{$counter}.{$value}
\n"; - -// Dev Actions -echo "

Developer Actions

"; -echo ""; -echo ""; - -if (isset($_POST['delete'])) { - $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$_POST['delete']}"); - $conn->commit(); -} -if (isset($_POST['match'])) { - match_and_merge_user_filesets($_POST['match']); - header("Location: {$filename}?id={$_POST['match']}"); -} - -echo ""; // Hidden - - -// Display history and logs -echo "

Fileset history

"; - -echo "\n"; -echo ""; -echo ""; -echo ""; -echo ""; - -$logs = $conn->query("SELECT `timestamp`, category, `text`, id FROM log -WHERE `text` REGEXP 'Fileset:{$id}' -ORDER BY `timestamp` DESC, id DESC"); - -while ($row = $logs->fetch_assoc()) { - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; -} - -while ($history_row = $history->fetch_assoc()) { - $logs = $conn->query("SELECT `timestamp`, category, `text`, id FROM log - WHERE `text` REGEXP 'Fileset:{$history_row['oldfileset']}' - AND `category` NOT REGEXP 'merge' - ORDER BY `timestamp` DESC, id DESC"); - - while ($row = $logs->fetch_assoc()) { - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; - echo "\n"; - } -} - -echo "
TimestampCategoryDescriptionLog ID
{$row['timestamp']}{$row['category']}{$row['text']}{$row['id']}
{$row['timestamp']}{$row['category']}{$row['text']}{$row['id']}
\n"; - -?> - diff --git a/fileset.py b/fileset.py new file mode 100644 index 00000000..497a9b63 --- /dev/null +++ b/fileset.py @@ -0,0 +1,930 @@ +from flask import Flask, request, render_template, redirect, url_for, render_template_string, jsonify, flash +import pymysql.cursors +import json +import re +import os +from user_fileset_functions import user_calc_key, file_json_to_array, user_insert_queue, user_insert_fileset, match_and_merge_user_filesets +from pagination import create_page +import difflib +from pymysql.converters import escape_string +from db_functions import find_matching_filesets, get_all_related_filesets, convert_log_text_to_links, user_integrity_check, db_connect,create_log +from collections import defaultdict + +app = Flask(__name__) + +secret_key = os.urandom(24) + +base_dir = os.path.dirname(os.path.abspath(__file__)) +config_path = os.path.join(base_dir, 'mysql_config.json') +with open(config_path) as f: + mysql_cred = json.load(f) + +conn = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + autocommit=False +) + +@app.route('/') +def index(): + html = """ + + + + + + +

Fileset Database

+

Fileset Actions

+ +

Logs

+ + + + """ + return render_template_string(html) + +@app.route('/fileset', methods=['GET', 'POST']) +def fileset(): + id = request.args.get('id', default=1, type=int) + widetable = request.args.get('widetable', default='partial', type=str) + # Load MySQL credentials from a JSON file + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + # Create a connection to the MySQL server + connection = pymysql.connect(host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + try: + with connection.cursor() as cursor: + # Get the minimum id from the fileset table + cursor.execute("SELECT MIN(id) FROM fileset") + min_id = cursor.fetchone()['MIN(id)'] + + # Get the id from the GET parameters, or use the minimum id if it's not provided + id = request.args.get('id', default=min_id, type=int) + + # Get the maximum id from the fileset table + cursor.execute("SELECT MAX(id) FROM fileset") + max_id = cursor.fetchone()['MAX(id)'] + + # Ensure the id is between the minimum and maximum id + id = max(min_id, min(id, max_id)) + + # Check if the id exists in the fileset table + cursor.execute(f"SELECT id FROM fileset WHERE id = {id}") + if cursor.rowcount == 0: + # If the id doesn't exist, get a new id from the history table + cursor.execute(f"SELECT fileset FROM history WHERE oldfileset = {id}") + id = cursor.fetchone()['fileset'] + + # Get the history for the current id + cursor.execute(f"SELECT `timestamp`, oldfileset, log FROM history WHERE fileset = {id} ORDER BY `timestamp`") + history = cursor.fetchall() + + # Display fileset details + html = f""" + + + + + + +

Fileset: {id}

+ + """ + html += f"" + html += f"" + html += f""" + + + + """ + cursor.execute(f"SELECT * FROM fileset WHERE id = {id}") + result = cursor.fetchone() + print(result) + html += "

Fileset details

" + html += "
\n" + if result['game']: + cursor.execute(f"SELECT game.name as 'game name', engineid, gameid, extra, platform, language FROM fileset JOIN game ON game.id = fileset.game JOIN engine ON engine.id = game.engine WHERE fileset.id = {id}") + result = {**result, **cursor.fetchone()} + else: + # result.pop('key', None) + # result.pop('status', None) + result.pop('delete', None) + + for column in result.keys(): + if column != 'id' and column != 'game': + html += f"\n" + + html += "\n" + for column, value in result.items(): + if column != 'id' and column != 'game': + html += f"" + html += "\n" + html += "
{column}
{value}
\n" + + # Files in the fileset + html += "

Files in the fileset

" + html += "
" + for k, v in request.args.items(): + if k != 'widetable': + html += f"" + if widetable == 'partial': + html += "" + html += "" + else: + html += "" + html += "" + html += "
" + + html += f"""
""" + # Table + html += "\n" + + sort = request.args.get('sort') + order = '' + md5_columns = ['md5-t-5000', 'md5-0', 'md5-5000', 'md5-1M'] + share_columns = ['name', 'size', 'checksum', 'detection', 'detection_type', 'timestamp'] + + if sort: + column = sort.split('-')[0] + valid_columns = share_columns + md5_columns + if column in valid_columns: + order = f"ORDER BY {column}" + if 'desc' in sort: + order += " DESC" + + columns_to_select = "file.id, name, size, checksum, detection, detection_type, `timestamp`" + columns_to_select += ", ".join(md5_columns) + print(f"SELECT file.id, name, size, checksum, detection, detection_type, `timestamp` FROM file WHERE fileset = {id} {order}") + cursor.execute(f"SELECT file.id, name, size, checksum, detection, detection_type, `timestamp` FROM file WHERE fileset = {id} {order}") + result = cursor.fetchall() + + all_columns = list(result[0].keys()) if result else [] + temp_set = set() + + if widetable == 'full': + file_ids = [file['id'] for file in result] + cursor.execute(f"SELECT file, checksum, checksize, checktype FROM filechecksum WHERE file IN ({','.join(map(str, file_ids))})") + checksums = cursor.fetchall() + + checksum_dict = {} + for checksum in checksums: + if checksum['checksize'] != 0: + key = f"{checksum['checktype']}-{checksum['checksize']}" + if checksum['file'] not in checksum_dict: + checksum_dict[checksum['file']] = {} + checksum_dict[checksum['file']][key] = checksum['checksum'] + temp_set.add(key) + + for index, file in enumerate(result): + if file['id'] in checksum_dict: + result[index].update(checksum_dict[file['id']]) + + all_columns.extend(list(temp_set)) + counter = 1 + # Generate table header + html += "\n" + html += "" # Checkbox column + sortable_columns = share_columns + list(temp_set) + + for column in sortable_columns: + if column not in ['id']: + vars = "&".join([f"{k}={v}" for k, v in request.args.items() if k != 'sort']) + sort_link = f"{column}" + if sort == column: + sort_link += "-desc" + html += f"\n" + html += "\n" + + # Generate table rows + for row in result: + html += "\n" + html += f"\n" + html += f"\n" # Checkbox for selecting file + for column in all_columns: + if column != 'id': + value = row.get(column, '') + if column == row.get('detection_type') and row.get('detection') == 1: + html += f"\n" + else: + html += f"\n" + html += "\n" + counter += 1 + + html += "
" # Numbering column + html += "Select{column}
{counter}.{value}{value}
\n" + html += "" + html += "
\n" + + # Generate the HTML for the developer actions + html += "

Developer Actions

" + html += f"" + html += f"" + + if 'delete' in request.form: + cursor.execute(f"UPDATE fileset SET `delete` = TRUE WHERE id = {request.form['delete']}") + connection.commit() + html += "

Fileset marked for deletion

" + + if 'match' in request.form: + match_and_merge_user_filesets(request.form['match']) + return redirect(url_for('fileset', id=request.form['match'])) + + # Generate the HTML for the fileset history + cursor.execute(f"SELECT `timestamp`, category, `text`, id FROM log WHERE `text` REGEXP 'Fileset:{id}' ORDER BY `timestamp` DESC, id DESC") + # cursor.execute(f"SELECT `timestamp`, fileset, oldfileset FROM history WHERE fileset = {id} ORDER BY `timestamp` DESC") + + logs = cursor.fetchall() + + html += "

Fileset history

" + html += "\n" + html += "\n" + html += "\n" + html += "\n" + html += "\n" + + related_filesets = get_all_related_filesets(id, conn) + + cursor.execute(f"SELECT * FROM history WHERE fileset IN ({','.join(map(str, related_filesets))}) OR oldfileset IN ({','.join(map(str, related_filesets))})") + history = cursor.fetchall() + print(f"History: {history}") + + for h in history: + cursor.execute(f"SELECT `timestamp`, category, `text`, id FROM log WHERE `text` LIKE 'Fileset:{h['oldfileset']}' ORDER BY `timestamp` DESC, id DESC") + logs = cursor.fetchall() + print(f"Logs: {logs}") + if h['fileset'] == h['oldfileset']: + continue + + if h['oldfileset'] == 0: + html += "\n" + html += f"\n" + html += f"\n" + html += f"\n" + # html += f"\n" + if h['log']: + cursor.execute(f"SELECT `text` FROM log WHERE id = {h['log']}") + log_text = cursor.fetchone()['text'] + log_text = convert_log_text_to_links(log_text) + html += f"\n" + else: + html += "\n" + html += "\n" + continue + + html += "\n" + html += f"\n" + html += f"\n" + html += f"\n" + # html += f"\n" + if h['log']: + cursor.execute(f"SELECT `text` FROM log WHERE id = {h['log']}") + log_text = cursor.fetchone()['text'] + log_text = convert_log_text_to_links(log_text) + html += f"\n" + else: + html += "\n" + html += "\n" + + html += "
TimestampCategoryDescriptionLog Text
{h['timestamp']}createCreated fileset Fileset {h['fileset']}Log {h['log']}Log {h['log']}: {log_text}No log available
{h['timestamp']}mergeFileset {h['oldfileset']} merged into fileset Fileset {h['fileset']}Log {h['log']}Log {h['log']}: {log_text}No log available
\n" + return render_template_string(html) + finally: + connection.close() + +@app.route('/fileset//match', methods=['GET']) +def match_fileset_route(id): + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + connection = pymysql.connect(host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + try: + with connection.cursor() as cursor: + cursor.execute(f"SELECT * FROM fileset WHERE id = {id}") + fileset = cursor.fetchone() + fileset['rom'] = [] + if not fileset: + return f"No fileset found with id {id}", 404 + + cursor.execute(f"SELECT file.id, name, size, checksum, detection, detection_type FROM file WHERE fileset = {id}") + result = cursor.fetchall() + file_ids = {} + for file in result: + file_ids[file['id']] = (file['name'], file['size']) + cursor.execute(f"SELECT file, checksum, checksize, checktype FROM filechecksum WHERE file IN ({','.join(map(str, file_ids.keys()))})") + + files = cursor.fetchall() + checksum_dict = defaultdict(lambda: {"name": "", "size": 0, "checksums": {}}) + + for i in files: + file_id = i["file"] + file_name, file_size = file_ids[file_id] + checksum_dict[file_name]["name"] = file_name + checksum_dict[file_name]["size"] = file_size + checksum_key = f"{i['checktype']}-{i['checksize']}" if i['checksize'] != 0 else i['checktype'] + checksum_dict[file_name]["checksums"][checksum_key] = i["checksum"] + + fileset["rom"] = [ + { + "name": value["name"], + "size": value["size"], + **value["checksums"] + } + for value in checksum_dict.values() + ] + + matched_map = find_matching_filesets(fileset, connection, fileset['status']) + + html = f""" + + + + + + +

Matched Filesets for Fileset: {id}

+ + + + + + + """ + + for fileset_id, match_count in matched_map.items(): + if fileset_id == id: + continue + cursor.execute(f"SELECT COUNT(file.id) FROM file WHERE fileset = {fileset_id}") + count = cursor.fetchone()['COUNT(file.id)'] + html += f""" + + + + + + + + """ + + html += "
Fileset IDMatch CountActions
{fileset_id}{len(match_count)} / {count}View Details +
+ + + +
+ 		+
+ +
+
" + return render_template_string(html) + finally: + connection.close() + +@app.route('/fileset//merge', methods=['GET', 'POST']) +def merge_fileset(id): + if request.method == 'POST': + search_query = request.form['search'] + + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + connection = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + try: + with connection.cursor() as cursor: + query = f""" + SELECT + fs.*, + g.name AS game_name, + g.engine AS game_engine, + g.platform AS game_platform, + g.language AS game_language, + g.extra AS extra + FROM + fileset fs + LEFT JOIN + game g ON fs.game = g.id + WHERE g.name LIKE '%{search_query}%' OR g.platform LIKE '%{search_query}%' OR g.language LIKE '%{search_query}%' + """ + cursor.execute(query) + results = cursor.fetchall() + + html = f""" + + + + + + +

Search Results for '{search_query}'

+
+ + +
+ + + """ + for result in results: + html += f""" + + + + + + + + + """ + html += "
IDGame NamePlatformLanguageExtraAction
{result['id']}{result['game_name']}{result['game_platform']}{result['game_language']}{result['extra']}Select
\n" + html += "\n" + + return render_template_string(html) + + finally: + connection.close() + + return ''' + + + + + + +

Search Fileset to Merge

+
+ + +
+ + + ''' + +@app.route('/fileset//merge/confirm', methods=['GET', 'POST']) +def confirm_merge(id): + target_id = request.args.get('target_id', type=int) if request.method == 'GET' else request.form.get('target_id') + + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + connection = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + try: + with connection.cursor() as cursor: + cursor.execute(f""" + SELECT + fs.*, + g.name AS game_name, + g.engine AS game_engine, + g.platform AS game_platform, + g.language AS game_language, + (SELECT COUNT(*) FROM file WHERE fileset = fs.id) AS file_count + FROM + fileset fs + LEFT JOIN + game g ON fs.game = g.id + WHERE + fs.id = {id} + """) + source_fileset = cursor.fetchone() + print(source_fileset) + cursor.execute(f""" + SELECT + fs.*, + g.name AS game_name, + g.engine AS game_engine, + g.platform AS game_platform, + g.language AS game_language, + (SELECT COUNT(*) FROM file WHERE fileset = fs.id) AS file_count + FROM + fileset fs + LEFT JOIN + game g ON fs.game = g.id + WHERE + fs.id = {target_id} + """) + target_fileset = cursor.fetchone() + + def highlight_differences(source, target): + diff = difflib.ndiff(source, target) + source_highlighted = "" + target_highlighted = "" + for d in diff: + if d.startswith('-'): + source_highlighted += f"{d[2:]}" + elif d.startswith('+'): + target_highlighted += f"{d[2:]}" + elif d.startswith(' '): + source_highlighted += d[2:] + target_highlighted += d[2:] + return source_highlighted, target_highlighted + + html = """ + + + + + + +

Confirm Merge

+ + + """ + for column in source_fileset.keys(): + source_value = str(source_fileset[column]) + target_value = str(target_fileset[column]) + if column == 'id': + html += f"" + continue + if source_value != target_value: + source_highlighted, target_highlighted = highlight_differences(source_value, target_value) + html += f"" + else: + html += f"" + + html += """ +
FieldSource FilesetTarget Fileset
{column}{source_value}{target_value}
{column}{source_highlighted}{target_highlighted}
{column}{source_value}{target_value}
+
+ + + +
+
+ +
+ + + """ + return render_template_string(html, source_fileset=source_fileset, target_fileset=target_fileset, id=id) + + finally: + connection.close() + +@app.route('/fileset//merge/execute', methods=['POST']) +def execute_merge(id, source=None, target=None): + source_id = request.form['source_id'] if not source else source + target_id = request.form['target_id'] if not target else target + + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + connection = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + try: + with connection.cursor() as cursor: + cursor.execute(f"SELECT * FROM fileset WHERE id = {source_id}") + source_fileset = cursor.fetchone() + cursor.execute(f"SELECT * FROM fileset WHERE id = {target_id}") + target_fileset = cursor.fetchone() + + if source_fileset['status'] == 'detection': + cursor.execute(f""" + UPDATE fileset SET + game = '{source_fileset['game']}', + status = '{source_fileset['status']}', + `key` = '{source_fileset['key']}', + megakey = '{source_fileset['megakey']}', + `timestamp` = '{source_fileset['timestamp']}' + WHERE id = {target_id} + """) + + cursor.execute(f"DELETE FROM file WHERE fileset = {target_id}") + + cursor.execute(f"SELECT * FROM file WHERE fileset = {source_id}") + source_files = cursor.fetchall() + + for file in source_files: + cursor.execute(f""" + INSERT INTO file (name, size, checksum, fileset, detection, `timestamp`) + VALUES ('{escape_string(file['name']).lower()}', '{file['size']}', '{file['checksum']}', {target_id}, {file['detection']}, NOW()) + """) + + cursor.execute("SELECT LAST_INSERT_ID() as file_id") + new_file_id = cursor.fetchone()['file_id'] + + cursor.execute(f"SELECT * FROM filechecksum WHERE file = {file['id']}") + file_checksums = cursor.fetchall() + + for checksum in file_checksums: + cursor.execute(f""" + INSERT INTO filechecksum (file, checksize, checktype, checksum) + VALUES ({new_file_id}, '{checksum['checksize']}', '{checksum['checktype']}', '{checksum['checksum']}') + """) + elif source_fileset['status'] in ['scan', 'dat']: + cursor.execute(f""" + UPDATE fileset SET + status = '{source_fileset['status'] if source_fileset['status'] != 'dat' else "partial"}', + `key` = '{source_fileset['key']}', + `timestamp` = '{source_fileset['timestamp']}' + WHERE id = {target_id} + """) + cursor.execute(f"SELECT * FROM file WHERE fileset = {source_id}") + source_files = cursor.fetchall() + + cursor.execute(f"SELECT * FROM file WHERE fileset = {target_id}") + target_files = cursor.fetchall() + + target_files_dict = {} + for target_file in target_files: + cursor.execute(f"SELECT * FROM filechecksum WHERE file = {target_file['id']}") + target_checksums = cursor.fetchall() + for checksum in target_checksums: + target_files_dict[checksum['checksum']] = target_file + + for source_file in source_files: + cursor.execute(f"SELECT * FROM filechecksum WHERE file = {source_file['id']}") + source_checksums = cursor.fetchall() + file_exists = False + for checksum in source_checksums: + print(checksum['checksum']) + if checksum['checksum'] in target_files_dict.keys(): + target_file = target_files_dict[checksum['checksum']] + source_file['detection'] = target_file['detection'] + + cursor.execute(f"DELETE FROM file WHERE id = {target_file['id']}") + file_exists = True + break + print(file_exists) + cursor.execute(f"""INSERT INTO file (name, size, checksum, fileset, detection, `timestamp`) VALUES ( + '{source_file['name']}', '{source_file['size']}', '{source_file['checksum']}', {target_id}, {source_file['detection']}, NOW())""") + new_file_id = cursor.lastrowid + for checksum in source_checksums: + # TODO: Handle the string + + cursor.execute("INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (%s, %s, %s, %s)", + (new_file_id, checksum['checksize'], f"{checksum['checktype']}-{checksum['checksize']}", checksum['checksum'])) + + cursor.execute(f""" + INSERT INTO history (`timestamp`, fileset, oldfileset) + VALUES (NOW(), {target_id}, {source_id}) + """) + + connection.commit() + + return redirect(url_for('fileset', id=target_id)) + + finally: + connection.close() + +@app.route('/fileset//mark_full', methods=['POST']) +def mark_as_full(id): + try: + conn = db_connect() + with conn.cursor() as cursor: + update_query = f"UPDATE fileset SET status = 'full' WHERE id = {id}" + cursor.execute(update_query) + create_log("Manual from Web", "Dev", f"Marked Fileset:{id} as full", conn) + conn.commit() + except Exception as e: + print(f"Error updating fileset status: {e}") + return jsonify({'error': 'Failed to mark fileset as full'}), 500 + finally: + conn.close() + + return redirect(f'/fileset?id={id}') + +@app.route('/validate', methods=['POST']) +def validate(): + error_codes = { + "unknown": -1, + "success": 0, + "empty": 2, + "no_metadata": 3, + } + + json_object = request.get_json() + + ip = request.remote_addr + ip = '.'.join(ip.split('.')[:3]) + '.X' + + game_metadata = {k: v for k, v in json_object.items() if k != 'files'} + + json_response = { + 'error': error_codes['success'], + 'files': [] + } + + if not game_metadata: + if not json_object.get('files'): + json_response['error'] = error_codes['empty'] + del json_response['files'] + json_response['status'] = 'empty_fileset' + return jsonify(json_response) + + json_response['error'] = error_codes['no_metadata'] + del json_response['files'] + json_response['status'] = 'no_metadata' + + fileset_id = user_insert_fileset(json_object, ip, conn) + json_response['fileset'] = fileset_id + print(f"Response: {json_response}") + return jsonify(json_response) + + matched_map = {} + missing_map = {} + extra_map = {} + + file_object = json_object['files'] + if not file_object: + json_response['error'] = error_codes['empty'] + json_response['status'] = 'empty_fileset' + return jsonify(json_response) + + try: + matched_map, missing_map, extra_map = user_integrity_check(json_object, ip, game_metadata) + except Exception as e: + json_response['error'] = -1 + json_response['status'] = 'processing_error' + json_response['fileset'] = 'unknown_fileset' + json_response['message'] = str(e) + print(f"Response: {json_response}") + return jsonify(json_response) + print(f"Matched: {matched_map}") + print(len(matched_map)) + if (len(matched_map) == 0): + json_response['error'] = error_codes['unknown'] + json_response['status'] = 'unknown_fileset' + json_response['fileset'] = 'unknown_fileset' + return jsonify(json_response) + matched_map = list(sorted(matched_map.items(), key=lambda x: len(x[1]), reverse=True))[0] + matched_id = matched_map[0] + # find the same id in the missing_map and extra_map + for fileset_id, count in missing_map.items(): + if fileset_id == matched_id: + missing_map = (fileset_id, count) + break + + for fileset_id, count in extra_map.items(): + if fileset_id == matched_id: + extra_map = (fileset_id, count) + break + + for file in matched_map[1]: + for key, value in file.items(): + if key == "name": + json_response['files'].append({'status': 'ok', 'fileset_id':matched_id, 'name': value}) + break + for file in missing_map[1]: + for key, value in file.items(): + if key == "name": + json_response['files'].append({'status': 'missing', 'fileset_id':matched_id, 'name': value}) + break + for file in extra_map[1]: + for key, value in file.items(): + if key == "name": + json_response['files'].append({'status': 'unknown_file', 'fileset_id':matched_id, 'name': value}) + break + print(f"Response: {json_response}") + return jsonify(json_response) + + +@app.route('/user_games_list') +def user_games_list(): + url = f"fileset_search?extra=&platform=&language=&megakey=&status=user" + return redirect(url) + +@app.route('/ready_for_review') +def ready_for_review(): + url = f"fileset_search?extra=&platform=&language=&megakey=&status=ReadyForReview" + return redirect(url) + +@app.route('/games_list') +def games_list(): + filename = "games_list" + records_table = "game" + select_query = """ + SELECT engineid, gameid, extra, platform, language, game.name, + status, fileset.id as fileset + FROM game + JOIN engine ON engine.id = game.engine + JOIN fileset ON game.id = fileset.game + """ + order = "ORDER BY gameid" + filters = { + "engineid": "engine", + "gameid": "game", + "extra": "game", + "platform": "game", + "language": "game", + "name": "game", + 'status': 'fileset' + } + mapping = { + 'engine.id': 'game.engine', + 'game.id': 'fileset.game', + } + return render_template_string(create_page(filename, 25, records_table, select_query, order, filters, mapping)) + +@app.route('/logs') +def logs(): + filename = "logs" + records_table = "log" + select_query = "SELECT id, `timestamp`, category, user, `text` FROM log" + order = "ORDER BY `timestamp` DESC, id DESC" + filters = { + 'id': 'log', + 'timestamp': 'log', + 'category': 'log', + 'user': 'log', + 'text': 'log' + } + return render_template_string(create_page(filename, 25, records_table, select_query, order, filters)) + +@app.route('/fileset_search') +def fileset_search(): + filename = "fileset_search" + records_table = "fileset" + select_query = """ + SELECT extra, platform, language, game.gameid, megakey, + status, fileset.id as fileset + FROM fileset + JOIN game ON game.id = fileset.game + """ + order = "ORDER BY fileset.id" + filters = { + "id": "fileset", + "gameid": "game", + "extra": "game", + "platform": "game", + "language": "game", + "megakey": "fileset", + "status": "fileset" + } + mapping = { + 'game.id': 'fileset.game', + } + return render_template_string(create_page(filename, 25, records_table, select_query, order, filters, mapping)) + +@app.route('/delete_files/', methods=['POST']) +def delete_files(id): + file_ids = request.form.getlist('file_ids') + if file_ids: + # Convert the list to comma-separated string for SQL + ids_to_delete = ",".join(file_ids) + connection = db_connect() + with connection.cursor() as cursor: + # SQL statements to delete related records + cursor.execute(f"DELETE FROM filechecksum WHERE file IN ({ids_to_delete})") + cursor.execute(f"DELETE FROM file WHERE id IN ({ids_to_delete})") + + # Commit the deletions + connection.commit() + return redirect(url_for('fileset', id=id)) + +if __name__ == '__main__': + app.secret_key = secret_key + app.run(debug=True, host='0.0.0.0') diff --git a/games_list.php b/games_list.php deleted file mode 100644 index c6950aad..00000000 --- a/games_list.php +++ /dev/null @@ -1,31 +0,0 @@ - table -$filters = array( - "engineid" => "engine", - "gameid" => "game", - "extra" => "game", - "platform" => "game", - "language" => "game", - "name" => "game", - "status" => "fileset" -); - -$mapping = array( - 'engine.id' => 'game.engine', - 'game.id' => 'fileset.game', -); - -create_page($filename, 25, $records_table, $select_query, $order, $filters, $mapping); -?> - diff --git a/include/db_functions.php b/include/db_functions.php deleted file mode 100644 index 81f21ffb..00000000 --- a/include/db_functions.php +++ /dev/null @@ -1,595 +0,0 @@ -set_charset('utf8mb4'); - $conn->autocommit(FALSE); - - // Check connection - if ($conn->connect_errno) { - die("Connect failed: " . $conn->connect_error); - } - - $conn->query("USE " . $dbname); - - return $conn; -} - -/** - * Retrieves the checksum and checktype of a given type + checksum - * eg: md5-5000 t:12345... -> 5000, md5-t, 12345... - */ -function get_checksum_props($checkcode, $checksum) { - $checksize = 0; - $checktype = $checkcode; - - if (strpos($checkcode, '-') !== false) { - $exploded_checkcode = explode('-', $checkcode); - $last = array_pop($exploded_checkcode); - if ($last == '1M' || is_numeric($last)) - $checksize = $last; - - $checktype = implode('-', $exploded_checkcode); - } - - // Detection entries have checktypes as part of the checksum prefix - if (strpos($checksum, ':') !== false) { - $prefix = explode(':', $checksum)[0]; - $checktype .= "-" . $prefix; - - $checksum = explode(':', $checksum)[1]; - } - - return array($checksize, $checktype, $checksum); -} - -/** - * Routine for inserting a game into the database - inserting into engine and - * game tables - */ -function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn) { - // Set @engine_last if engine already present in table - $exists = false; - if ($res = $conn->query(sprintf("SELECT id FROM engine WHERE engineid = '%s'", $engineid))) { - if ($res->num_rows > 0) { - $exists = true; - $conn->query(sprintf("SET @engine_last = '%d'", $res->fetch_array()[0])); - } - } - - // Insert into table if not present - if (!$exists) { - $query = sprintf("INSERT INTO engine (name, engineid) - VALUES ('%s', '%s')", mysqli_real_escape_string($conn, $engine_name), $engineid); - $conn->query($query); - $conn->query("SET @engine_last = LAST_INSERT_ID()"); - } - - // Insert into game - $query = sprintf("INSERT INTO game (name, engine, gameid, extra, platform, language) - VALUES ('%s', @engine_last, '%s', '%s', '%s', '%s')", mysqli_real_escape_string($conn, $title), - $gameid, mysqli_real_escape_string($conn, $extra), $platform, $lang); - $conn->query($query); - $conn->query("SET @game_last = LAST_INSERT_ID()"); -} - -function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_text, $conn, $ip = '') { - $status = $detection ? "detection" : $src; - $game = "NULL"; - $key = $key == "" ? "NULL" : "'{$key}'"; - $megakey = $megakey == "" ? "NULL" : "'{$megakey}'"; - - if ($detection) { - $status = "detection"; - $game = "@game_last"; - } - - // Check if key/megakey already exists, if so, skip insertion (no quotes on purpose) - if ($detection) - $existing_entry = $conn->query("SELECT id FROM fileset WHERE `key` = {$key}"); - else - $existing_entry = $conn->query("SELECT id FROM fileset WHERE megakey = {$megakey}"); - - if ($existing_entry->num_rows > 0) { - $existing_entry = $existing_entry->fetch_array()[0]; - $conn->query("SET @fileset_last = {$existing_entry}"); - - $category_text = "Uploaded from {$src}"; - $log_text = "Duplicate of Fileset:{$existing_entry}, {$log_text}"; - if ($src == 'user') - $log_text = "Duplicate of Fileset:{$existing_entry}, from user IP {$ip}, {$log_text}"; - - $user = 'cli:' . get_current_user(); - create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text)); - - if (!$detection) - return false; - - $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last) - WHERE id = {$existing_entry}"); - $conn->query("UPDATE fileset SET status = 'detection' - WHERE id = {$existing_entry} AND status = 'obsolete'"); - $conn->query("DELETE FROM game WHERE id = @game_last"); - return false; - } - - // $game and $key should not be parsed as a mysql string, hence no quotes - $query = "INSERT INTO fileset (game, status, src, `key`, megakey, `timestamp`) - VALUES ({$game}, '{$status}', '{$src}', {$key}, {$megakey}, FROM_UNIXTIME(@fileset_time_last))"; - $conn->query($query); - $conn->query("SET @fileset_last = LAST_INSERT_ID()"); - - $category_text = "Uploaded from {$src}"; - $fileset_last = $conn->query("SELECT @fileset_last")->fetch_array()[0]; - $log_text = "Created Fileset:{$fileset_last}, {$log_text}"; - if ($src == 'user') - $log_text = "Created Fileset:{$fileset_last}, from user IP {$ip}, {$log_text}"; - - $user = 'cli:' . get_current_user(); - create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text)); - $conn->query("INSERT INTO transactions (`transaction`, fileset) VALUES ({$transaction}, {$fileset_last})"); - - return true; -} - -/** - * Routine for inserting a file into the database, inserting into all - * required tables - * $file is an associated array (the contents of 'rom') - * If checksum of the given checktype doesn't exists, silently fails - */ -function insert_file($file, $detection, $src, $conn) { - // Find full md5, or else use first checksum value - $checksum = ""; - $checksize = 5000; - if (isset($file["md5"])) { - $checksum = $file["md5"]; - } - else { - foreach ($file as $key => $value) { - if (strpos($key, "md5") !== false) { - list($checksize, $checktype, $checksum) = get_checksum_props($key, $value); - break; - } - } - } - - $query = sprintf("INSERT INTO file (name, size, checksum, fileset, detection) - VALUES ('%s', '%s', '%s', @fileset_last, %d)", mysqli_real_escape_string($conn, $file["name"]), - $file["size"], $checksum, $detection); - $conn->query($query); - - if ($detection) - $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL"); - $conn->query("SET @file_last = LAST_INSERT_ID()"); -} - -function insert_filechecksum($file, $checktype, $conn) { - if (!array_key_exists($checktype, $file)) - return; - - $checksum = $file[$checktype]; - list($checksize, $checktype, $checksum) = get_checksum_props($checktype, $checksum); - - $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum) - VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $checksum); - $conn->query($query); -} - -/** - * Delete filesets marked for deletion - */ -function delete_filesets($conn) { - $query = "DELETE FROM fileset WHERE `delete` == TRUE"; - $conn->query($query); -} - -/** - * Create an entry to the log table on each call of db_insert() or - * populate_matching_games() - */ -function create_log($category, $user, $text) { - $conn = db_connect(); - $conn->query(sprintf("INSERT INTO log (`timestamp`, category, user, `text`) - VALUES (FROM_UNIXTIME(%d), '%s', '%s', '%s')", time(), $category, $user, $text)); - $log_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0]; - - if (!$conn->commit()) - echo "Creating log failed\n"; - - return $log_last; -} - -/** - * Calculate `key` value as md5("name:title:...:engine:file1:size:md5:file2:...") - */ -function calc_key($fileset) { - $key_string = ""; - - foreach ($fileset as $key => $value) { - if ($key == 'engineid' || $key == 'gameid' || $key == 'rom') - continue; - - $key_string .= ':' . $value; - } - - $files = $fileset['rom']; - foreach ($files as $file) { - foreach ($file as $key => $value) { - $key_string .= ':' . $value; - } - } - - $key_string = trim($key_string, ':'); - return md5($key_string); -} - -/** - * Calculate `megakey` value as md5("file1:size:md5:file2:...") - */ -function calc_megakey($files) { - $key_string = ""; - foreach ($files as $file) { - foreach ($file as $key => $value) { - $key_string .= ':' . $value; - } - } - $key_string = trim($key_string, ':'); - return md5($key_string); -} - -/** - * Insert values from the associated array into the DB - * They will be inserted under gameid NULL as the game itself is unconfirmed - */ -function db_insert($data_arr) { - $header = $data_arr[0]; - $game_data = $data_arr[1]; - $resources = $data_arr[2]; - $filepath = $data_arr[3]; - - $conn = db_connect(); - - /** - * Author can be: - * scummvm -> Detection Entries - * scanner -> CLI scanner tool in python - * _anything else_ -> DAT file - */ - $author = $header["author"]; - $version = $header["version"]; - - /** - * status can be: - * detection -> Detection entries (source of truth) - * user -> Submitted by users via ScummVM, unmatched (Not used in the parser) - * scan -> Submitted by cli/scanner, unmatched - * dat -> Submitted by DAT, unmatched - * partialmatch -> Submitted by DAT, matched - * fullmatch -> Submitted by cli/scanner, matched - * obsolete -> Detection entries that are no longer part of the detection set - */ - $src = ""; - if ($author == "scan" || $author == "scummvm") - $src = $author; - else - $src = "dat"; - - $detection = ($src == "scummvm"); - $status = $detection ? "detection" : $src; - - // Set timestamp of fileset insertion - $conn->query(sprintf("SET @fileset_time_last = %d", time())); - - // Create start log entry - $transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1; - - $category_text = "Uploaded from {$src}"; - $log_text = sprintf("Started loading DAT file, size %d, author '%s', version %s. - State '%s'. Transaction: %d", - $filepath, filesize($filepath), $author, $version, $status, $transaction_id); - - $user = 'cli:' . get_current_user(); - create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text)); - - foreach ($game_data as $fileset) { - if ($detection) { - $engine_name = $fileset["engine"]; - $engineid = $fileset["sourcefile"]; - $gameid = $fileset["name"]; - $title = $fileset["title"]; - $extra = $fileset["extra"]; - $platform = $fileset["platform"]; - $lang = $fileset["language"]; - - insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn); - } - elseif ($src == "dat") - if (isset($fileset['romof']) && isset($resources[$fileset['romof']])) - $fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]); - - $key = $detection ? calc_key($fileset) : ""; - $megakey = !$detection ? calc_megakey($fileset['rom']) : ""; - $log_text = sprintf("size %d, author '%s', version %s. - State '%s'.", - filesize($filepath), $author, $version, $status); - - if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) { - foreach ($fileset["rom"] as $file) { - insert_file($file, $detection, $src, $conn); - foreach ($file as $key => $value) { - if ($key != "name" && $key != "size") - insert_filechecksum($file, $key, $conn); - } - } - } - } - - if ($detection) - $conn->query("UPDATE fileset SET status = 'obsolete' - WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last) - AND status = 'detection'"); - - $fileset_insertion_count = $conn->query("SELECT COUNT(fileset) from transactions WHERE `transaction` = {$transaction_id}")->fetch_array()[0]; - $category_text = "Uploaded from {$src}"; - $log_text = sprintf("Completed loading DAT file, filename '%s', size %d, author '%s', version %s. - State '%s'. Number of filesets: %d. Transaction: %d", - $filepath, filesize($filepath), $author, $version, $status, $fileset_insertion_count, $transaction_id); - - if (!$conn->commit()) - echo "Inserting failed\n"; - else { - $user = 'cli:' . get_current_user(); - create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text)); - } -} - -/** - * Compare 2 dat filesets to find if they are equivalent or not - */ -function compare_filesets($id1, $id2, $conn) { - $fileset1 = $conn->query("SELECT name, size, checksum - FROM file WHERE fileset = '{$id1}'")->fetch_array(); - $fileset2 = $conn->query("SELECT name, size, checksum - FROM file WHERE fileset = '{$id2}'")->fetch_array(); - - // Sort filesets on checksum - usort($fileset1, function ($a, $b) { - return $a[2] <=> $b[2]; - }); - usort($fileset2, function ($a, $b) { - return $a[2] <=> $b[2]; - }); - - if (count($fileset1) != count($fileset2)) - return false; - - for ($i = 0; $i < count($fileset1); $i++) { - // If checksums do not match - if ($fileset1[2] != $fileset2[2]) - return false; - } - - return True; -} - -/** - * Return fileset statuses that can be merged with set of given status - * eg: scan and dat -> detection - * fullmatch -> partialmatch, detection - */ -function status_to_match($status) { - $order = array("detection", "dat", "scan", "partialmatch", "fullmatch", "user"); - return array_slice($order, 0, array_search($status, $order)); -} - -/** - * Detects games based on the file descriptions in $dat_arr - * Compares the files with those in the detection entries table - * $game_files consists of both the game ( ) and resources ( ) parts - */ -function find_matching_game($game_files) { - $matching_games = array(); // All matching games - $matching_filesets = array(); // All filesets containing one file from $game_files - $matches_count = 0; // Number of files with a matching detection entry - - $conn = db_connect(); - - foreach ($game_files as $file) { - $checksum = $file[1]; - - $query = "SELECT file.fileset as file_fileset - FROM filechecksum - JOIN file ON filechecksum.file = file.id - WHERE filechecksum.checksum = '{$checksum}' AND file.detection = TRUE"; - $records = $conn->query($query)->fetch_all(); - - // If file is not part of detection entries, skip it - if (count($records) == 0) - continue; - - $matches_count++; - foreach ($records as $record) - array_push($matching_filesets, $record[0]); - } // Check if there is a fileset_id that is present in all results - foreach (array_count_values($matching_filesets) as $key => $value) { - $count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file - JOIN fileset ON file.fileset = fileset.id - WHERE fileset.id = '%s'", $key))->fetch_array()[0]; - - // We use < instead of != since one file may have more than one entry in the fileset - // We see this in Drascula English version, where one entry is duplicated - if ($value < $matches_count || $value < $count_files_in_fileset) - continue; - - $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform, - language, `key`, src, fileset.id as fileset - FROM game - JOIN fileset ON fileset.game = game.id - JOIN engine ON engine.id = game.engine - WHERE fileset.id = '%s'", $key)); - - array_push($matching_games, $records->fetch_array()); - } - - if (count($matching_games) != 1) - return $matching_games; - - // Check the current fileset priority with that of the match - $records = $conn->query(sprintf("SELECT id FROM fileset, ({$query}) AS res - WHERE id = file_fileset AND - status IN ('%s')", implode("', '", status_to_match($game_files[3])))); - - // If priority order is correct - if ($records->num_rows != 0) - return $matching_games; - - if (compare_filesets($matching_games[0]['fileset'], $game_files[0][0], $conn)) { - $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0][0]}"); - return array(); - } - - return $matching_games; -} - -/** - * Merge two filesets without duplicating files - * Used after matching an unconfirmed fileset with a detection entry - */ -function merge_filesets($detection_id, $dat_id) { - $conn = db_connect(); - - $detection_files = $conn->query(sprintf("SELECT DISTINCT(filechecksum.checksum), checksize, checktype - FROM filechecksum JOIN file on file.id = filechecksum.file - WHERE fileset = '%d'", $detection_id))->fetch_all(); - - foreach ($detection_files as $file) { - $checksum = $file[0]; - $checksize = $file[1]; - $checktype = $file[2]; - - // Delete original detection entry so newly matched fileset is the only fileset for game - $conn->query(sprintf("DELETE FROM file - WHERE checksum = '%s' AND fileset = %d LIMIT 1", $checksum, $detection_id)); - - // Mark files present in the detection entries - $conn->query(sprintf("UPDATE file - JOIN filechecksum ON filechecksum.file = file.id - SET detection = TRUE, - checksize = %d, - checktype = '%s' - WHERE fileset = '%d' AND filechecksum.checksum = '%s'", - $checksize, $checktype, $dat_id, $checksum)); - } - - // Add fileset pair to history ($dat_id is the new fileset for $detection_id) - $conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset) - VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id)); - $history_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0]; - - $conn->query("UPDATE history SET fileset = {$dat_id} WHERE fileset = {$detection_id}"); - - // Delete original fileset - $conn->query("DELETE FROM fileset WHERE id = {$detection_id}"); - - if (!$conn->commit()) - echo "Error merging filesets\n"; - - return $history_last; -} - -/** - * (Attempt to) match fileset that have fileset.game as NULL - * This will delete the original detection fileset and replace it with the newly - * matched fileset - */ -function populate_matching_games() { - $conn = db_connect(); - - // Getting unmatched filesets - $unmatched_filesets = array(); - - $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status - FROM fileset - JOIN file ON file.fileset = fileset.id - JOIN filechecksum ON file.id = filechecksum.file - WHERE fileset.game IS NULL AND status != 'user'"); - $unmatched_files = $unmatched_files->fetch_all(); - - // Splitting them into different filesets - for ($i = 0; $i < count($unmatched_files); $i++) { - $cur_fileset = $unmatched_files[$i][0]; - $temp = array(); - while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) { - array_push($temp, $unmatched_files[$i]); - $i++; - } - array_push($unmatched_filesets, $temp); - } - - foreach ($unmatched_filesets as $fileset) { - $matching_games = find_matching_game($fileset); - - if (count($matching_games) != 1) // If there is no match/non-unique match - continue; - - $matched_game = $matching_games[0]; - - // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch) - $status = $fileset[0][2]; - if ($fileset[0][2] == "dat") - $status = "partialmatch"; - elseif ($fileset[0][2] == "scan") - $status = "fullmatch"; - - // Convert NULL values to string with value NULL for printing - $matched_game = array_map(function ($val) { - return (is_null($val)) ? "NULL" : $val; - }, $matched_game); - - $category_text = "Matched from {$fileset[0][2]}"; - $log_text = "Matched game {$matched_game['engineid']}: - {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']} - variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}."; - - // Updating the fileset.game value to be $matched_game["id"] - $query = sprintf("UPDATE fileset - SET game = %d, status = '%s', `key` = '%s' - WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]); - - $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]); - - if ($conn->query($query)) { - $user = 'cli:' . get_current_user(); - - // Merge log - create_log("Fileset merge", $user, - mysqli_real_escape_string($conn, "Merged Fileset:{$matched_game['fileset']} and Fileset:{$fileset[0][0]}")); - - // Matching log - $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user, - mysqli_real_escape_string($conn, $log_text)); - - // Add log id to the history table - $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}"); - } - - if (!$conn->commit()) - echo "Updating matched games failed\n"; - } -} - - -?> - diff --git a/include/pagination.php b/include/pagination.php deleted file mode 100644 index 7269137b..00000000 --- a/include/pagination.php +++ /dev/null @@ -1,255 +0,0 @@ -\n"; -echo "\n"; -echo "\n"; - -/** - * Return a string denoting which two columns link two tables - */ -function get_join_columns($table1, $table2, $mapping) { - foreach ($mapping as $primary => $foreign) { - $primary = explode('.', $primary); - $foreign = explode('.', $foreign); - if (($primary[0] == $table1 && $foreign[0] == $table2) || - ($primary[0] == $table2 && $foreign[0] == $table1)) - return "{$primary[0]}.{$primary[1]} = {$foreign[0]}.{$foreign[1]}"; - } - - echo "No primary-foreign key mapping provided. Filter is invalid"; -} - -function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) { - $mysql_cred = json_decode(file_get_contents(__DIR__ . '/../mysql_config.json'), true); - $servername = $mysql_cred["servername"]; - $username = $mysql_cred["username"]; - $password = $mysql_cred["password"]; - $dbname = $mysql_cred["dbname"]; - - // Create connection - mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); - $conn = new mysqli($servername, $username, $password); - $conn->set_charset('utf8mb4'); - $conn->autocommit(FALSE); - - // Check connection - if ($conn->connect_errno) { - die("Connect failed: " . $conn->connect_error); - } - - $conn->query("USE " . $dbname); - - // If there exist get variables that are for filtering - $_GET = array_filter($_GET); - if (isset($_GET['sort'])) { - $column = $_GET['sort']; - $column = explode('-', $column); - $order = "ORDER BY {$column[0]}"; - - if (strpos($_GET['sort'], 'desc') !== false) - $order .= " DESC"; - } - - if (array_diff(array_keys($_GET), array('page', 'sort'))) { - $condition = "WHERE "; - $tables = array(); - foreach ($_GET as $key => $value) { - if ($key == 'page' || $key == 'sort' || $value == '') - continue; - - array_push($tables, $filters[$key]); - $condition .= $condition != "WHERE " ? " AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'"; - } - if ($condition == "WHERE ") - $condition = ""; - - // If more than one table is to be searched - $from_query = $records_table; - if (count($tables) > 1 || $tables[0] != $records_table) - for ($i = 0; $i < count($tables); $i++) { - if ($tables[$i] == $records_table) - continue; - - $from_query .= sprintf(" JOIN %s ON %s", $tables[$i], get_join_columns($records_table, $tables[$i], $mapping)); - } - - $num_of_results = $conn->query( - "SELECT COUNT({$records_table}.id) FROM {$from_query} {$condition}")->fetch_array()[0]; - } - // If $records_table has a JOIN (multiple tables) - elseif (preg_match("/JOIN/", $records_table) !== false) { - $first_table = explode(" ", $records_table)[0]; - $num_of_results = $conn->query("SELECT COUNT({$first_table}.id) FROM {$records_table}")->fetch_array()[0]; - } - else { - $num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0]; - } - $num_of_pages = ceil($num_of_results / $results_per_page); - if ($num_of_results == 0) { - echo "No results for given filters"; - return; - } - - if (!isset($_GET['page'])) { - $page = 1; - } - else { - $page = max(1, min($_GET['page'], $num_of_pages)); - } - - $offset = ($page - 1) * $results_per_page; - - // If there exist get variables that are for filtering - if (array_diff(array_keys($_GET), array('page'))) { - $condition = "WHERE "; - foreach ($_GET as $key => $value) { - $value = mysqli_real_escape_string($conn, $value); - if (!isset($filters[$key])) - continue; - - $condition .= $condition != "WHERE " ? "AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'"; - } - if ($condition == "WHERE ") - $condition = ""; - - $query = "{$select_query} {$condition} {$order} LIMIT {$results_per_page} OFFSET {$offset}"; - } - else { - $query = "{$select_query} {$order} LIMIT {$results_per_page} OFFSET {$offset}"; - } - $result = $conn->query($query); - - - // Table - echo "
"; - echo "\n"; - - $counter = $offset + 1; - while ($row = $result->fetch_assoc()) { - if ($counter == $offset + 1) { // If it is the first run of the loop - if (count($filters) > 0) { - echo ""; - foreach (array_keys($row) as $key) { - if (!isset($filters[$key])) { - echo "\n"; - } - echo ""; - echo ""; - } - - echo "\n"; - else - echo "\n"; - } - } - - if ($filename == 'games_list.php' || $filename == 'user_games_list.php') - echo "\n"; - else - echo "\n"; - echo "\n"; - foreach ($row as $key => $value) { - if ($key == 'fileset') - continue; - - // Add links to fileset in logs table - $matches = array(); - if (preg_match("/Fileset:(\d+)/", $value, $matches, PREG_OFFSET_CAPTURE)) { - $value = substr($value, 0, $matches[0][1]) . - "{$matches[0][0]}" . - substr($value, $matches[0][1] + strlen($matches[0][0])); - } - - echo "\n"; - } - echo "\n"; - - $counter++; - } - - echo "
"; - continue; - } - - // Filter textbox - $filter_value = isset($_GET[$key]) ? $_GET[$key] : ""; - - echo "
\n"; // Numbering column - foreach (array_keys($row) as $key) { - if ($key == 'fileset') - continue; - - // Preserve GET variables - $vars = ""; - foreach ($_GET as $k => $v) { - if ($k == 'sort' && $v == $key) - $vars .= "&{$k}={$v}-desc"; - elseif ($k != 'sort') - $vars .= "&{$k}={$v}"; - } - - if (strpos($vars, "&sort={$key}") === false) - echo "{$key}{$key}
{$counter}.{$value}
\n"; - echo "
\n"; - - // Preserve GET variables - $vars = ""; - foreach ($_GET as $key => $value) { - if ($key == 'page') - continue; - $vars .= "&{$key}={$value}"; - } - - // Navigation elements - if ($num_of_pages > 1) { - echo "
\n"; - - // Preserve GET variables on form submit - foreach ($_GET as $key => $value) { - if ($key == 'page') - continue; - - $key = htmlspecialchars($key); - $value = htmlspecialchars($value); - if ($value != "") - echo ""; - } - - echo "\n"; - - echo "
\n"; - } - -} -?> - diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php deleted file mode 100644 index b160436c..00000000 --- a/include/user_fileset_functions.php +++ /dev/null @@ -1,153 +0,0 @@ - $value) { - if ($key != 'checksums') { - $key_string .= ':' . $value; - continue; - } - - foreach ($value as $checksum_pair) - $key_string .= ':' . $checksum_pair->checksum; - } - } - $key_string = trim($key_string, ':'); - - return md5($key_string); -} - -function file_json_to_array($file_json_object) { - $res = array(); - - foreach ($file_json_object as $key => $value) { - if ($key != 'checksums') { - $res[$key] = $value; - continue; - } - - foreach ($value as $checksum_pair) - $res[$checksum_pair->type] = $checksum_pair->checksum; - } - - return $res; -} - -function user_insert_queue($user_fileset, $conn) { - $query = sprintf("INSERT INTO queue (time, notes, fileset, ticketid, userid, commit) - VALUES (%d, NULL, @fileset_last, NULL, NULL, NULL)", time()); - - $conn->query($query); -} - -function user_insert_fileset($user_fileset, $ip, $conn) { - $src = 'user'; - $detection = false; - $key = ''; - $megakey = user_calc_key($user_fileset); - $transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1; - $log_text = "from user submitted files"; - $conn = db_connect(); - - // Set timestamp of fileset insertion - $conn->query(sprintf("SET @fileset_time_last = %d", time())); - - if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn, $ip)) { - foreach ($user_fileset as $file) { - $file = file_json_to_array($file); - - insert_file($file, $detection, $src, $conn); - foreach ($file as $key => $value) { - if ($key != "name" && $key != "size") - insert_filechecksum($file, $key, $conn); - } - } - } - - $fileset_id = $conn->query("SELECT @fileset_last")->fetch_array()[0]; - $conn->commit(); - return $fileset_id; -} - - -/** - * (Attempt to) match fileset that have fileset.game as NULL - * This will delete the original detection fileset and replace it with the newly - * matched fileset - */ -function match_and_merge_user_filesets($id) { - $conn = db_connect(); - - // Getting unmatched filesets - $unmatched_filesets = array(); - - $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status - FROM fileset - JOIN file ON file.fileset = fileset.id - JOIN filechecksum ON file.id = filechecksum.file - WHERE status = 'user' AND fileset.id = {$id}"); - $unmatched_files = $unmatched_files->fetch_all(); - - // Splitting them into different filesets - for ($i = 0; $i < count($unmatched_files); $i++) { - $cur_fileset = $unmatched_files[$i][0]; - $temp = array(); - while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) { - array_push($temp, $unmatched_files[$i]); - $i++; - } - array_push($unmatched_filesets, $temp); - } - - foreach ($unmatched_filesets as $fileset) { - $matching_games = find_matching_game($fileset); - - if (count($matching_games) != 1) // If there is no match/non-unique match - continue; - - $matched_game = $matching_games[0]; - - $status = 'fullmatch'; - - // Convert NULL values to string with value NULL for printing - $matched_game = array_map(function ($val) { - return (is_null($val)) ? "NULL" : $val; - }, $matched_game); - - $category_text = "Matched from {$fileset[0][2]}"; - $log_text = "Matched game {$matched_game['engineid']}: - {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']} - variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}."; - - // Updating the fileset.game value to be $matched_game["id"] - $query = sprintf("UPDATE fileset - SET game = %d, status = '%s', `key` = '%s' - WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]); - - $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]); - - if ($conn->query($query)) { - $user = 'cli:' . get_current_user(); - - // Merge log - create_log("Fileset merge", $user, - mysqli_real_escape_string($conn, "Merged Fileset:{$matched_game['fileset']} and Fileset:{$fileset[0][0]}")); - - // Matching log - $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user, - mysqli_real_escape_string($conn, $log_text)); - - // Add log id to the history table - $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}"); - } - - if (!$conn->commit()) - echo "Updating matched games failed\n"; - } -} - - -?> - diff --git a/index.php b/index.php deleted file mode 100644 index f76a86a1..00000000 --- a/index.php +++ /dev/null @@ -1,15 +0,0 @@ - diff --git a/logs.php b/logs.php deleted file mode 100644 index 53b538ab..00000000 --- a/logs.php +++ /dev/null @@ -1,20 +0,0 @@ - 'log', - 'timestamp' => 'log', - 'category' => 'log', - 'user' => 'log', - 'text' => 'log' -); - -create_page($filename, 25, $records_table, $select_query, $order, $filters); -?> - diff --git a/megadata.py b/megadata.py new file mode 100644 index 00000000..d4893910 --- /dev/null +++ b/megadata.py @@ -0,0 +1,38 @@ +import os +import time +import compute_hash + +class Megadata: + def __init__(self, file_path): + self.file_path = file_path + self.hash = self.calculate_hash(file_path) + self.size = os.path.getsize(file_path) + self.creation_time = os.path.getctime(file_path) + self.modification_time = os.path.getmtime(file_path) + + def calculate_hash(self, file_path): + pass + + def __eq__(self, other): + return (self.hash == other.hash and + self.size == other.size and + self.creation_time == other.creation_time and + self.modification_time == other.modification_time) + + +def record_megadata(directory): + file_megadata = {} + for root, _, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + file_megadata[file_path] = Megadata(file_path) + return file_megadata + +def check_for_updates(old_megadata, current_directory): + current_megadata = record_megadata(current_directory) + updates = [] + for old_path, old_data in old_megadata.items(): + for current_path, current_data in current_megadata.items(): + if old_data == current_data and old_path != current_path: + updates.append((old_path, current_path)) + return updates \ No newline at end of file diff --git a/mod_actions.php b/mod_actions.php deleted file mode 100644 index 845b285b..00000000 --- a/mod_actions.php +++ /dev/null @@ -1,28 +0,0 @@ -\n"; -echo "\n"; -echo "\n"; - - -// Dev Tools -echo "

Developer Moderation Tools

"; -echo ""; -echo "
"; -echo ""; -echo "
"; -echo ""; - -if (isset($_POST['delete'])) { -} -if (isset($_POST['match'])) { - // merge_user_filesets(); -} - -echo ""; // Hidden -?> - diff --git a/pagination.py b/pagination.py new file mode 100644 index 00000000..57caead3 --- /dev/null +++ b/pagination.py @@ -0,0 +1,212 @@ +from flask import Flask, request, render_template_string +import pymysql +import json +import re +import os + +app = Flask(__name__) + +stylesheet = 'style.css' +jquery_file = 'https://code.jquery.com/jquery-3.7.0.min.js' +js_file = 'js_functions.js' + +def get_join_columns(table1, table2, mapping): + for primary, foreign in mapping.items(): + primary = primary.split('.') + foreign = foreign.split('.') + if (primary[0] == table1 and foreign[0] == table2) or (primary[0] == table2 and foreign[0] == table1): + return f"{primary[0]}.{primary[1]} = {foreign[0]}.{foreign[1]}" + return "No primary-foreign key mapping provided. Filter is invalid" + +def create_page(filename, results_per_page, records_table, select_query, order, filters={}, mapping={}): + base_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(base_dir, 'mysql_config.json') + with open(config_path) as f: + mysql_cred = json.load(f) + + conn = pymysql.connect( + host=mysql_cred["servername"], + user=mysql_cred["username"], + password=mysql_cred["password"], + db=mysql_cred["dbname"], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + with conn.cursor() as cursor: + # Handle sorting + sort = request.args.get('sort') + if sort: + column = sort.split('-') + order = f"ORDER BY {column[0]}" + if 'desc' in sort: + order += " DESC" + + if set(request.args.keys()).difference({'page', 'sort'}): + condition = "WHERE " + tables = [] + for key, value in request.args.items(): + if key in ['page', 'sort'] or value == '': + continue + tables.append(filters[key]) + if value == '': + value = '.*' + condition += f" AND {filters[key]}.{key} REGEXP '{value}'" if condition != "WHERE " else f"{filters[key]}.{key} REGEXP '{value}'" + + if condition == "WHERE ": + condition = "" + + # Handle multiple tables + from_query = records_table + if len(tables) > 1 or (tables and tables[0] != records_table): + for table in tables: + if table == records_table: + continue + from_query += f" JOIN {table} ON {get_join_columns(records_table, table, mapping)}" + cursor.execute(f"SELECT COUNT({records_table}.id) AS count FROM {from_query} {condition}") + num_of_results = cursor.fetchone()['count'] + + elif "JOIN" in records_table: + first_table = records_table.split(" ")[0] + cursor.execute(f"SELECT COUNT({first_table}.id) FROM {records_table}") + num_of_results = cursor.fetchone()[f'COUNT({first_table}.id)'] + else: + cursor.execute(f"SELECT COUNT(id) FROM {records_table}") + num_of_results = cursor.fetchone()['COUNT(id)'] + + num_of_pages = (num_of_results + results_per_page - 1) // results_per_page + print(f"Num of results: {num_of_results}, Num of pages: {num_of_pages}") + if num_of_results == 0: + return "No results for given filters" + + page = int(request.args.get('page', 1)) + page = max(1, min(page, num_of_pages)) + offset = (page - 1) * results_per_page + + # Fetch results + if set(request.args.keys()).difference({'page'}): + condition = "WHERE " + for key, value in request.args.items(): + if key not in filters: + continue + + value = pymysql.converters.escape_string(value) + if value == '': + value = '.*' + condition += f" AND {filters[key]}.{key} REGEXP '{value}'" if condition != "WHERE " else f"{filters[key]}.{key} REGEXP '{value}'" + + if condition == "WHERE ": + condition = "" + + query = f"{select_query} {condition} {order} LIMIT {results_per_page} OFFSET {offset}" + else: + query = f"{select_query} {order} LIMIT {results_per_page} OFFSET {offset}" + cursor.execute(query) + results = cursor.fetchall() + + # Generate HTML + html = f""" + + + + + + +
+ +""" + if not results: + return "No results for given filters" + if results: + if filters: + html += "" + for key in results[0].keys(): + if key not in filters: + html += "" + continue + filter_value = request.args.get(key, "") + html += f"" + html += "" + + html += "" + for key in results[0].keys(): + if key == 'fileset': + continue + vars = "&".join([f"{k}={v}" for k, v in request.args.items() if k != 'sort']) + sort = request.args.get('sort', '') + if sort == key: + vars += f"&sort={key}-desc" + else: + vars += f"&sort={key}" + html += f"" + + counter = offset + 1 + for row in results: + if counter == offset + 1: # If it is the first run of the loop + if filters: + html += "" + for key in row.keys(): + if key not in filters: + html += "" + continue + + # Filter textbox + filter_value = request.args.get(key, "") + + if records_table != "log": + fileset_id = row['fileset'] + html += f"\n" + html += f"\n" + else: + html += "\n" + html += f"\n" + + for key, value in row.items(): + if key == 'fileset': + continue + + # Add links to fileset in logs table + if isinstance(value, str): + matches = re.search(r"Fileset:(\d+)", value) + if matches: + fileset_id = matches.group(1) + fileset_text = matches.group(0) + value = value.replace(fileset_text, f"{fileset_text}") + + html += f"\n" + html += "\n" + + counter += 1 + + html += "
{key}
{counter}.
{counter}.{value}
" + + # Pagination + vars = "&".join([f"{k}={v}" for k, v in request.args.items() if k != 'page']) + + if num_of_pages > 1: + html += "
" + for key, value in request.args.items(): + if key != 'page': + html += f"" + html += "
" + + return html diff --git a/schema.py b/schema.py new file mode 100644 index 00000000..8e75085a --- /dev/null +++ b/schema.py @@ -0,0 +1,233 @@ +import json +import pymysql +import random +import string +from datetime import datetime +import os + +# Load MySQL credentials +base_dir = os.path.dirname(os.path.abspath(__file__)) +config_path = os.path.join(base_dir, 'mysql_config.json') +with open(config_path) as f: + mysql_cred = json.load(f) + +servername = mysql_cred["servername"] +username = mysql_cred["username"] +password = mysql_cred["password"] +dbname = mysql_cred["dbname"] + +# Create connection +conn = pymysql.connect( + host=servername, + user=username, + password=password, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + autocommit=False +) + +# Check connection +if conn is None: + print("Error connecting to MySQL") + exit(1) + +cursor = conn.cursor() + +# Create database +sql = f"CREATE DATABASE IF NOT EXISTS {dbname}" +cursor.execute(sql) + +# Use database +cursor.execute(f"USE {dbname}") + +# Create tables +tables = { + "engine": """ + CREATE TABLE IF NOT EXISTS engine ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200), + engineid VARCHAR(100) NOT NULL + ) + """, + "game": """ + CREATE TABLE IF NOT EXISTS game ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200), + engine INT NOT NULL, + gameid VARCHAR(100) NOT NULL, + extra VARCHAR(200), + platform VARCHAR(30), + language VARCHAR(10), + FOREIGN KEY (engine) REFERENCES engine(id) + ) + """, + "file": """ + CREATE TABLE IF NOT EXISTS file ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200) NOT NULL, + size BIGINT NOT NULL, + checksum VARCHAR(64) NOT NULL, + fileset INT NOT NULL, + detection BOOLEAN NOT NULL, + FOREIGN KEY (fileset) REFERENCES fileset(id) ON DELETE CASCADE + ) + """, + "filechecksum": """ + CREATE TABLE IF NOT EXISTS filechecksum ( + id INT AUTO_INCREMENT PRIMARY KEY, + file INT NOT NULL, + checksize VARCHAR(10) NOT NULL, + checktype VARCHAR(10) NOT NULL, + checksum VARCHAR(64) NOT NULL, + FOREIGN KEY (file) REFERENCES file(id) ON DELETE CASCADE + ) + """, + "queue": """ + CREATE TABLE IF NOT EXISTS queue ( + id INT AUTO_INCREMENT PRIMARY KEY, + time TIMESTAMP NOT NULL, + notes varchar(300), + fileset INT, + userid INT NOT NULL, + commit VARCHAR(64) NOT NULL, + FOREIGN KEY (fileset) REFERENCES fileset(id) + ) + """, + "fileset": """ + CREATE TABLE IF NOT EXISTS fileset ( + id INT AUTO_INCREMENT PRIMARY KEY, + game INT, + status VARCHAR(20), + src VARCHAR(20), + `key` VARCHAR(64), + `megakey` VARCHAR(64), + `delete` BOOLEAN DEFAULT FALSE NOT NULL, + `timestamp` TIMESTAMP NOT NULL, + detection_size INT, + FOREIGN KEY (game) REFERENCES game(id) + ) + """, + "log": """ + CREATE TABLE IF NOT EXISTS log ( + id INT AUTO_INCREMENT PRIMARY KEY, + `timestamp` TIMESTAMP NOT NULL, + category VARCHAR(100) NOT NULL, + user VARCHAR(100) NOT NULL, + `text` varchar(300) + ) + """, + "history": """ + CREATE TABLE IF NOT EXISTS history ( + id INT AUTO_INCREMENT PRIMARY KEY, + `timestamp` TIMESTAMP NOT NULL, + fileset INT NOT NULL, + oldfileset INT NOT NULL, + log INT + ) + """, + "transactions": """ + CREATE TABLE IF NOT EXISTS transactions ( + id INT AUTO_INCREMENT PRIMARY KEY, + `transaction` INT NOT NULL, + fileset INT NOT NULL + ) + """ +} + +for table, definition in tables.items(): + try: + cursor.execute(definition) + print(f"Table '{table}' created successfully") + except pymysql.Error as err: + print(f"Error creating '{table}' table: {err}") + +# Create indices +indices = { + "detection": "CREATE INDEX detection ON file (detection)", + "checksum": "CREATE INDEX checksum ON filechecksum (checksum)", + "engineid": "CREATE INDEX engineid ON engine (engineid)", + "key": "CREATE INDEX fileset_key ON fileset (`key`)", + "status": "CREATE INDEX status ON fileset (status)", + "fileset": "CREATE INDEX fileset ON history (fileset)" +} + +try: + cursor.execute("ALTER TABLE file ADD COLUMN detection_type VARCHAR(20);") +except: + # if aleady exists, change the length of the column + cursor.execute("ALTER TABLE file MODIFY COLUMN detection_type VARCHAR(20);") + +try: + cursor.execute("ALTER TABLE file ADD COLUMN `timestamp` TIMESTAMP NOT NULL;") +except: + # if aleady exists, change the length of the column + cursor.execute("ALTER TABLE file MODIFY COLUMN `timestamp` TIMESTAMP NOT NULL;") + +try: + cursor.execute("ALTER TABLE fileset ADD COLUMN `user_count` INT;") +except: + # if aleady exists, change the length of the column + cursor.execute("ALTER TABLE fileset MODIFY COLUMN `user_count` INT;") + +try: + cursor.execute("ALTER TABLE file ADD COLUMN punycode_name VARCHAR(200);") +except: + cursor.execute("ALTER TABLE file MODIFY COLUMN punycode_name VARCHAR(200);") + +try: + cursor.execute("ALTER TABLE file ADD COLUMN encoding_type VARCHAR(20) DEFAULT 'UTF-8';") +except: + cursor.execute("ALTER TABLE file MODIFY COLUMN encoding_type VARCHAR(20) DEFAULT 'UTF-8';") + +for index, definition in indices.items(): + try: + cursor.execute(definition) + print(f"Created index for '{index}'") + except pymysql.Error as err: + print(f"Error creating index for '{index}': {err}") + +# Insert random data into tables +def random_string(length=10): + return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) + +def insert_random_data(): + for _ in range(1000): + # Insert data into engine + cursor.execute("INSERT INTO engine (name, engineid) VALUES (%s, %s)", (random_string(), random_string())) + + # Insert data into game + cursor.execute("INSERT INTO game (name, engine, gameid, extra, platform, language) VALUES (%s, %s, %s, %s, %s, %s)", + (random_string(), 1, random_string(), random_string(), random_string(), random_string())) + + # Insert data into fileset + cursor.execute("INSERT INTO fileset (game, status, src, `key`, `megakey`, `timestamp`, detection_size) VALUES (%s, %s, %s, %s, %s, %s, %s)", + (1, 'user', random_string(), random_string(), random_string(), datetime.now(), random.randint(1, 100))) + + # Insert data into file + cursor.execute("INSERT INTO file (name, size, checksum, fileset, detection) VALUES (%s, %s, %s, %s, %s)", + (random_string(), random.randint(1000, 10000), random_string(), 1, True)) + + # Insert data into filechecksum + cursor.execute("INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (%s, %s, %s, %s)", + (1, random_string(), random_string(), random_string())) + + # Insert data into queue + cursor.execute("INSERT INTO queue (time, notes, fileset, userid, commit) VALUES (%s, %s, %s, %s, %s)", + (datetime.now(), random_string(), 1, random.randint(1, 100), random_string())) + + # Insert data into log + cursor.execute("INSERT INTO log (`timestamp`, category, user, `text`) VALUES (%s, %s, %s, %s)", + (datetime.now(), random_string(), random_string(), random_string())) + + # Insert data into history + cursor.execute("INSERT INTO history (`timestamp`, fileset, oldfileset, log) VALUES (%s, %s, %s, %s)", + (datetime.now(), 1, 2, 1)) + + # Insert data into transactions + cursor.execute("INSERT INTO transactions (`transaction`, fileset) VALUES (%s, %s)", + (random.randint(1, 100), 1)) +# for testing locally +# insert_random_data() + +conn.commit() +conn.close() \ No newline at end of file diff --git a/static/style.css b/static/style.css new file mode 100644 index 00000000..1c9e599c --- /dev/null +++ b/static/style.css @@ -0,0 +1,113 @@ +:root { + --primary-color: #27b5e8; + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; +} + +td, th { + padding-inline: 5px; +} + +tr:nth-child(even) {background-color: #f2f2f2;} +tr {background-color: white;} + +tr:hover {background-color: #ddd;} +tr.games_list:hover {cursor: pointer;} + +tr.filter:hover {background-color:inherit;} +td.filter {text-align: center;} + +th { + padding-top: 5px; + padding-bottom: 5px; + text-align: center; + background-color: var(--primary-color); + color: white; +} + +th a { + color: white; + text-decoration: none; /* no underline */ +} + +button { + color: white; + padding: 6px 12px; + border-radius: 10px; + transition: background-color 0.1s; + background-color: var(--primary-color); + border: 1px solid var(--primary-color); +} + +button:hover { + background-color: #29afe0; +} +button:active { + background-color: #1a95c2; +} + +input[type=submit] { + color: white; + padding: 6px 12px; + border-radius: 10px; + transition: background-color 0.1s; + background-color: var(--primary-color); + border: 1px solid var(--primary-color); +} + +input[type=submit]:hover { + background-color: #29afe0; +} +input[type=submit]:active { + background-color: #1a95c2; +} + +input[type=text], select { + width: 25%; + height: 38px; + padding: 6px 12px; + margin: 0px 8px; + display: inline-block; + border: 1px solid #ccc; + border-radius: 4px; + box-sizing: border-box; +} + +input[type=text].filter { + width: 80%; +} + +.pagination { + display: inline-block; + align-self: center; +} + +.pagination .more { + color: black; + float: left; + padding: 15px 10px; +} + +.pagination a { + color: black; + float: left; + padding: 8px 16px; + text-decoration: none; + transition: background-color 0.3s; + border: 1px solid #ddd; +} + +.pagination a.active { + color: white; + background-color: var(--primary-color); + border: 1px solid var(--primary-color); +} + +.pagination a:hover:not(.active) { + background-color: #ddd; +} + +form { + padding: 0px; + margin: 0px; + display: inline; +} diff --git a/user_fileset_functions.py b/user_fileset_functions.py new file mode 100644 index 00000000..80e0c1a9 --- /dev/null +++ b/user_fileset_functions.py @@ -0,0 +1,157 @@ +import hashlib +import time +from db_functions import db_connect, insert_fileset, insert_file, insert_filechecksum, find_matching_game, merge_filesets, create_log, calc_megakey +import getpass +import pymysql + +def user_calc_key(user_fileset): + key_string = "" + for file in user_fileset: + for key, value in file.items(): + if key != 'checksums': + key_string += ':' + str(value) + continue + for checksum_pair in value: + key_string += ':' + checksum_pair['checksum'] + key_string = key_string.strip(':') + return hashlib.md5(key_string.encode()).hexdigest() + +def file_json_to_array(file_json_object): + res = {} + for key, value in file_json_object.items(): + if key != 'checksums': + res[key] = value + continue + for checksum_pair in value: + res[checksum_pair['type']] = checksum_pair['checksum'] + return res + +def user_insert_queue(user_fileset, conn): + query = f"INSERT INTO queue (time, notes, fileset, ticketid, userid, commit) VALUES ({int(time.time())}, NULL, @fileset_last, NULL, NULL, NULL)" + + with conn.cursor() as cursor: + cursor.execute(query) + conn.commit() + +def user_insert_fileset(user_fileset, ip, conn): + src = 'user' + detection = False + key = '' + megakey = calc_megakey(user_fileset) + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + transaction_id = cursor.fetchone()['MAX(`transaction`)'] + 1 + log_text = "from user submitted files" + cursor.execute("SET @fileset_time_last = %s", (int(time.time()))) + if insert_fileset(src, detection, key, megakey, transaction_id, log_text, conn, ip): + for file in user_fileset['files']: + file = file_json_to_array(file) + insert_file(file, detection, src, conn) + for key, value in file.items(): + if key not in ["name", "size"]: + insert_filechecksum(file, key, conn) + cursor.execute("SELECT @fileset_last") + fileset_id = cursor.fetchone()['@fileset_last'] + conn.commit() + return fileset_id + +def match_and_merge_user_filesets(id): + conn = db_connect() + + # Getting unmatched filesets + unmatched_filesets = [] + + with conn.cursor() as cursor: + cursor.execute(f"SELECT fileset.id, filechecksum.checksum, src, status FROM fileset JOIN file ON file.fileset = fileset.id JOIN filechecksum ON file.id = filechecksum.file WHERE status = 'user' AND fileset.id = {id}") + unmatched_files = cursor.fetchall() + + # Splitting them into different filesets + i = 0 + while i < len(unmatched_files): + cur_fileset = unmatched_files[i][0] + temp = [] + while i < len(unmatched_files) and cur_fileset == unmatched_files[i][0]: + temp.append(unmatched_files[i]) + i += 1 + unmatched_filesets.append(temp) + + for fileset in unmatched_filesets: + matching_games = find_matching_game(fileset) + + if len(matching_games) != 1: # If there is no match/non-unique match + continue + + matched_game = matching_games[0] + + status = 'full' + + # Convert NULL values to string with value NULL for printing + matched_game = {k: 'NULL' if v is None else v for k, v in matched_game.items()} + + category_text = f"Matched from {fileset[0][2]}" + log_text = f"Matched game {matched_game['engineid']}:\n{matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']}\nvariant {matched_game['key']}. State {status}. Fileset:{fileset[0][0]}." + + # Updating the fileset.game value to be $matched_game["id"] + query = f"UPDATE fileset SET game = {matched_game['id']}, status = '{status}', `key` = '{matched_game['key']}' WHERE id = {fileset[0][0]}" + + history_last = merge_filesets(matched_game["fileset"], fileset[0][0]) + + if cursor.execute(query): + user = f'cli:{getpass.getuser()}' + + # Merge log + create_log("Fileset merge", user, pymysql.escape_string(conn, f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}")) + + # Matching log + log_last = create_log(pymysql.escape_string(conn, category_text), user, pymysql.escape_string(conn, log_text)) + + # Add log id to the history table + cursor.execute(f"UPDATE history SET log = {log_last} WHERE id = {history_last}") + + if not conn.commit(): + print("Updating matched games failed") + with conn.cursor() as cursor: + cursor.execute(""" + SELECT fileset.id, filechecksum.checksum, src, status + FROM fileset + JOIN file ON file.fileset = fileset.id + JOIN filechecksum ON file.id = filechecksum.file + WHERE status = 'user' AND fileset.id = %s + """, (id,)) + unmatched_files = cursor.fetchall() + + unmatched_filesets = [] + cur_fileset = None + temp = [] + for file in unmatched_files: + if cur_fileset is None or cur_fileset != file['id']: + if temp: + unmatched_filesets.append(temp) + cur_fileset = file['id'] + temp = [] + temp.append(file) + if temp: + unmatched_filesets.append(temp) + + for fileset in unmatched_filesets: + matching_games = find_matching_game(fileset) + if len(matching_games) != 1: + continue + matched_game = matching_games[0] + status = 'full' + matched_game = {k: ("NULL" if v is None else v) for k, v in matched_game.items()} + category_text = f"Matched from {fileset[0]['src']}" + log_text = f"Matched game {matched_game['engineid']}: {matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']} variant {matched_game['key']}. State {status}. Fileset:{fileset[0]['id']}." + query = """ + UPDATE fileset + SET game = %s, status = %s, `key` = %s + WHERE id = %s + """ + history_last = merge_filesets(matched_game["fileset"], fileset[0]['id']) + with conn.cursor() as cursor: + cursor.execute(query, (matched_game["id"], status, matched_game["key"], fileset[0]['id'])) + user = 'cli:' + getpass.getuser() + create_log("Fileset merge", user, f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0]['id']}") + log_last = create_log(category_text, user, log_text) + cursor.execute("UPDATE history SET log = %s WHERE id = %s", (log_last, history_last)) + conn.commit() diff --git a/user_games_list.php b/user_games_list.php deleted file mode 100644 index e6706b47..00000000 --- a/user_games_list.php +++ /dev/null @@ -1,32 +0,0 @@ - table -$filters = array( - "engineid" => "engine", - "gameid" => "game", - "extra" => "game", - "platform" => "game", - "language" => "game", - "name" => "game", - "status" => "fileset" -); - -$mapping = array( - 'engine.id' => 'game.engine', - 'game.id' => 'fileset.game', -); - -create_page($filename, 200, $records_table, $select_query, $order, $filters, $mapping); -?> -