Merge pull request #1300 from mailwatch/migrate-to-dbd-mariadb

Migrate MailScanner perl scripts to DBD::MariaDB
mailwatch · Jul 31, 2024 · 0b58f3e · 0b58f3e
2 parents 5e41f3a + 2bafb80
commit 0b58f3e
Show file tree

Hide file tree

Showing 5 changed files with 73 additions and 120 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,9 @@
 - Update htmlpurifier library to 4.17.0
 - Avoid usage of deprecated utf8_encode function in viewpart
 
+### Compatibility
+- Migrate MailScanner perl script to DBD:MariaDB
+
 ## 1.2.23
 ### Added
 - Support for handling the `uopz` extension to prevent the application from breaking due to disabled `exit` calls by `uopz`.

diff --git a/MailScanner_perl_scripts/MailWatch.pm b/MailScanner_perl_scripts/MailWatch.pm
@@ -2,11 +2,11 @@
 # MailWatch for MailScanner
 # Copyright (C) 2003-2011  Steve Freegard (steve@freegard.name)
 # Copyright (C) 2011  Garrod Alwood (garrod.alwood@lorodoes.com)
-# Copyright (C) 2014-2021  MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
+# Copyright (C) 2014-2024  MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
 #
 #   Custom Module MailWatch
 #
-#   Version 1.61
+#   Version 1.7
 #
 # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
@@ -31,6 +31,7 @@ package MailScanner::CustomConfig;
 
 use strict;
 use DBI;
+use DBD::MariaDB;
 use utf8;
 use Sys::Hostname;
 use Storable(qw[freeze thaw]);
@@ -46,7 +47,7 @@ use Sys::Syslog;
 use vars qw($VERSION);
 
 ### The package version, both in 1.23 style *and* usable by MakeMaker:
-$VERSION = substr q$Revision: 1.5 $, 10;
+$VERSION = '1.7';
 
 # Trace settings - uncomment this to debug
 #DBI->trace(2,'/tmp/dbitrace.log');
@@ -119,20 +120,20 @@ sub InitMailWatchLogging {
 sub CheckSQLVersion {
     # Prevent Logger from dying if connection fails
     eval {
-        $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
             $db_user, $db_pass,
-            { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
         );
     };
     if ($@ || !$dbh) {
         LogMessage("warn", "Unable to initialise database connection: $DBI::errstr");
         close(SERVER);
         return 1;
     }
-    $SQLversion = $dbh->{mysql_serverversion};
+    $SQLversion = $dbh->{mariadb_serverversion};
     $dbh->disconnect;
-    return $SQLversion;
 
+    return $SQLversion;
 }
 
 sub LogMessage {
@@ -177,29 +178,17 @@ sub InitDB {
        return 1;
     }
 
-    if (CheckSQLVersion() >= 50503 ) {
-        eval { $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-            $db_user, $db_pass,
-            { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
-        );
-        };
-        if ($@ || !$dbh) {
-            LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
-            return 1;
-        }
-        $dbh->do('SET NAMES utf8mb4');
-    } else {
-        eval { $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
+    eval { $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
             $db_user, $db_pass,
-            { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
         );
-        };
-        if ($@ || !$dbh) {
-            LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
-            return 1;
-        }
-        $dbh->do('SET NAMES utf8');
+    };
+    if ($@ || !$dbh) {
+        LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
+        return 1;
     }
+    $dbh->do('SET NAMES utf8mb4');
+
     $sth = $dbh->prepare("INSERT INTO maillog (timestamp, id, size, from_address, from_domain, to_address, to_domain, subject, clientip, archive, isspam, ishighspam, issaspam, isrblspam, spamwhitelisted, spamblacklisted, sascore, spamreport, virusinfected, nameinfected, otherinfected, report, ismcp, ishighmcp, issamcp, mcpwhitelisted, mcpblacklisted, mcpsascore, mcpreport, hostname, date, time, headers, quarantined, rblspamreport, token, messageid) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
     if (!$sth) {
         LogMessage('warn', "Error: $DBI::errstr" );

diff --git a/MailScanner_perl_scripts/SQLBlackWhiteList.pm b/MailScanner_perl_scripts/SQLBlackWhiteList.pm
@@ -2,11 +2,11 @@
 # MailWatch for MailScanner
 # Copyright (C) 2003-2011  Steve Freegard (steve@freegard.name)
 # Copyright (C) 2011  Garrod Alwood (garrod.alwood@lorodoes.com)
-# Copyright (C) 2014-2021  MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
+# Copyright (C) 2014-2024  MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
 #
 #   Custom Module SQLBlackWhiteList
 #
-#   Version 1.6
+#   Version 1.7
 #
 # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
@@ -39,9 +39,10 @@ use vars qw($VERSION);
 #use Data::Dumper;
 
 ### The package version, both in 1.23 style *and* usable by MakeMaker:
-$VERSION = substr q$Revision: 1.5 $, 10;
+$VERSION = '1.7';
 
 use DBI;
+use DBD::MariaDB;
 my (%Whitelist, %Blacklist);
 my ($wtime, $btime);
 my ($dbh);
@@ -61,23 +62,23 @@ my ($db_pass) = mailwatch_get_db_password();
 # Get refresh time from from 00MailWatchConf.pm
 my ($bwl_refresh_time) =  mailwatch_get_BWL_refresh_time();
 
-# Check MySQL version
+# Check MySQL/MariaDB version
 sub CheckSQLVersion {
     # Prevent dying from failed db connection
     eval { 
-        $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
             $db_user, $db_pass,
-            { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
         );
     };
     if ($@ || !$dbh) {
         MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList:: Unable to initialise database connection: %s", $DBI::errstr);
         return 1;
     }
-    $SQLversion = $dbh->{mysql_serverversion};
+    $SQLversion = $dbh->{mariadb_serverversion};
     $dbh->disconnect;
-    return $SQLversion;
 
+    return $SQLversion;
 }
 
 #
@@ -135,39 +136,25 @@ sub CreateList {
     my ($type, $BlackWhite) = @_;
     my ($sql, $to_address, $from_address, $count, $filter);
 
-    # Check if MySQL is >= 5.3.3
+    # Check if MySQL/MariaDB is >= 5.3.3
     my $version = CheckSQLVersion();
 
     # Database connection failed, so return 0 (count) and exit early
     if ($version == 1) {
         return 0;
     }
 
-    if ($version >= 50503 ) {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8mb4');
-    } else {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8');
+    eval {
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
+            $db_user, $db_pass,
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
+        );
+    };
+    if ($@ || !$dbh) {
+        MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
+        return 0;
     }
+    $dbh->do('SET NAMES utf8mb4');
 
     # Uncommet the folloging line when debugging SQLBlackWhiteList.pm
     #MailScanner::Log::WarnLog("MailWatch: DEBUG SQLBlackWhiteList: CreateList: %s", Dumper($BlackWhite));

diff --git a/MailScanner_perl_scripts/SQLSpamSettings.pm b/MailScanner_perl_scripts/SQLSpamSettings.pm
@@ -2,11 +2,11 @@
 # MailWatch for MailScanner
 # Copyright (C) 2003-2011  Steve Freegard (steve@freegard.name)
 # Copyright (C) 2011  Garrod Alwood (garrod.alwood@lorodoes.com)
-# Copyright (C) 2014-2021  MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
+# Copyright (C) 2014-2024  MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
 #
 #   Custom Module SQLSpamSettings
 #
-#   Version 1.6
+#   Version 1.7
 #
 # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
@@ -41,9 +41,10 @@ no  strict 'subs'; # Allow bare words for parameter %'s
 use vars qw($VERSION);
 
 ### The package version, both in 1.23 style *and* usable by MakeMaker:
-$VERSION = substr q$Revision: 1.5 $, 10;
+$VERSION = '1.7';
 
 use DBI;
+use DBD::MariaDB;
 my ($dbh);
 my ($sth);
 my ($SQLversion);
@@ -64,20 +65,22 @@ my ($db_pass) = mailwatch_get_db_password();
 # Get refresh time from from 00MailWatchConf.pm
 my ($ss_refresh_time) =  mailwatch_get_SS_refresh_time();
 
-# Check MySQL version
+# Check MySQL/MariaDB version
 sub CheckSQLVersion {
+    # Prevent dying from failed db connection
     eval {
-        $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
             $db_user, $db_pass,
-            { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
         );
     };
     if ($@ || !$dbh) {
         MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: Unable to initialise database connection: %s", $DBI::errstr);
         return 1;
     }
-    $SQLversion = $dbh->{mysql_serverversion};
+    $SQLversion = $dbh->{mariadb_serverversion};
     $dbh->disconnect;
+
     return $SQLversion;
 }
 
@@ -169,39 +172,25 @@ sub CreateScoreList
     my ($type, $UserList) = @_;
     my ($sql, $username, $count);
 
-    # Check if MySQL is >= 5.3.3
+    # Check if MySQL/MariaDB is >= 5.3.3
     my $version = CheckSQLVersion();
 
     # Cannot get SQL version, bail out with count of 0
     if ($version == 1) {
         return 0;
     }
 
-    if ($version >= 50503 ) {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8mb4');
-    } else {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8');
+    eval {
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
+            $db_user, $db_pass,
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
+        );
+    };
+    if ($@ || !$dbh) {
+        MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
+        return 0;
     }
+    $dbh->do('SET NAMES utf8mb4');
 
     $sql = "SELECT username, $type FROM users WHERE $type > 0";
     $sth = $dbh->prepare($sql);
@@ -228,32 +217,17 @@ sub CreateNoScanList
     my ($type, $NoScanList) = @_;
     my ($sql, $username, $count);
 
-    # Check if MySQL is >= 5.3.3
-    if (CheckSQLVersion() >= 50503 ) {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8mb4');
-    } else {
-        eval {
-            $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
-                $db_user, $db_pass,
-                { PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
-            );
-        };
-        if ($@ || !$dbh) {
-            MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
-            return 0;
-        }
-        $dbh->do('SET NAMES utf8');
+    eval {
+        $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
+            $db_user, $db_pass,
+            { PrintError => 0, AutoCommit => 1, RaiseError => 1 }
+        );
+    };
+    if ($@ || !$dbh) {
+        MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
+        return 0;
     }
+    $dbh->do('SET NAMES utf8mb4');
 
     $sql = "SELECT username, $type FROM users WHERE $type > 0";
     $sth = $dbh->prepare($sql);

diff --git a/Remote_DB.md b/Remote_DB.md
@@ -10,7 +10,7 @@ This document presumes that you will have a server acting as a database with PHP
     mysql> GRANT FILE ON *.* TO mailwatch IDENTIFIED BY '<password>';
     mysql> flush privileges;
 
-3)  On each MailScanner gateway, you'll need to make sure that the mysql client, perl, perl DBI and perl DBD-Mysql (4.032 or higher) are installed: see https://docs.mailwatch.org/install/getting-started.html
+3)  On each MailScanner gateway, you'll need to make sure that the mysql client, perl, perl DBI and perl DBD-MariaDB are installed: see https://docs.mailwatch.org/install/getting-started.html
 
 4)  From one of the MailScanner gateway, verify you can connect to the db:  
     % mysql mailscanner -u mailwatch -h <db_hostname> -p  
@@ -19,6 +19,6 @@ This document presumes that you will have a server acting as a database with PHP
 
 5)  On each MailScanner gateway continue following the install instructions at https://docs.mailwatch.org/install/installing.html#create-a-mysql-user-and-password--set-up-mailscanner-for-sql-logging
 
-7)  On each MailWatch system set RPC_ALLOWED_CLIENTS in conf.php to a list of IP addresses of each MailWatch system.
+6)  On each MailWatch system set RPC_ALLOWED_CLIENTS in conf.php to a list of IP addresses of each MailWatch system.
 
-8) (Optional) If you want a combined display of the number of mails in the mail queues also set RPC_REMOTE_SERVER in conf.php
+7) (Optional) If you want a combined display of the number of mails in the mail queues also set RPC_REMOTE_SERVER in conf.php