dave/Data_Warehouse
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Dave Umrysh 2021-03-19 11:26:42 -06:00
commit f57e9f79ea
2 changed files with 144 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
getDBLog.php Normal file
View File

@ -0,0 +1,35 @@
<?php
ini_set("memory_limit","8024M");
header('Access-Control-Allow-Origin: *');
date_default_timezone_set('UTC');
$api_key = "cC7d2DHDnEwJTwiHLakhyjZGKLyWR";
if( isset($_REQUEST['apiKey']) && $_REQUEST['apiKey'] == $api_key && isset($_REQUEST['timestamp']) && is_numeric($_REQUEST['timestamp']) ) {
$connect = mysqli_connect ('example-cluster-1.cluster-xxxxxxxxxx.ca-central-1.rds.amazonaws.com', 'dbuser', 'password');
mysqli_select_db ( $connect,'mysql');
$output = '{ "message": "", "data": { "transactions": [';
// Get the data from the current log
$query = 'select UNIX_TIMESTAMP(event_time) as event_time,argument from mysql.general_log where (upper(argument) like "%INSERT %" or upper(argument) like "%UPDATE %" or upper(argument) like "%CREATE %") and argument NOT LIKE "%mysql.rds_heartbeat2%" AND argument NOT LIKE "%INFORMATION_SCHEMA.GLOBAL_STATUS%"';
$result = mysqli_query ( $connect, $query);
while ($thedata = mysqli_fetch_assoc($result)) {
$output = $output . '{"event_time" : "' . str_replace('"', '\"', $thedata["event_time"]) . '","argument" : "' . str_replace('"', '\"', str_replace('\"', '\\\"', str_replace("\'", "'", $thedata["argument"]))) . '"},';
}
$output = rtrim($output, ",");
$output = $output . ']}}';
// Rotate the general log table
$query = 'CALL mysql.rds_rotate_general_log';
$result = mysqli_query ( $connect, $query);
echo $output;
}else{
echo '{ "message": "Invalid post variables", "data": {}}';
}
?>

109
warehouse_get.php Normal file
View File

@ -0,0 +1,109 @@
<?php
// if started from commandline, wrap parameters to $_POST and $_GET
if (!isset($_SERVER["HTTP_HOST"]) && isset($argv[1])) {
parse_str($argv[1], $_REQUEST);
}
$api_key = "cC7d2DHDnEwJTwiHLakhyjZGKLyWR";
$web_hook_room = "https://example.com/_matrix/maubot/plugin/botincomingwebhook/webhook/r0?room=%21xxxxxxxxxxxxx:example.com";
$web_hook_secret = "s5dXEqBThKiuwQj8etALv6A";
if( isset($_REQUEST['apiKey']) && $_REQUEST['apiKey'] == $api_key ) {
$web_result = "";
$records_inserted = 0;
try {
$host = '127.0.0.1';
$user = 'dbuser';
$pass = 'password';
$database = 'datawarehouse';
// connect to the mysql database server.
$connect = mysqli_connect ( $host, $user, $pass ) ;
if ( ! $connect )
{
exit();
}
mysqli_select_db ( $connect,$database);
// Get the new queries
$web_result = file_get_contents("https://example.com/getDBLog.php?apiKey=".$api_key."&timestamp=1614301200");
// Save a copy for debugging
file_put_contents(dirname(__FILE__) . "/output.json", $web_result);
// Parse the queries
$web_result = iconv('UTF-8', 'UTF-8//IGNORE', utf8_encode($web_result));
$json_object = json_decode($web_result);
switch (json_last_error()) {
case JSON_ERROR_NONE:
echo ' - No errors';
break;
case JSON_ERROR_DEPTH:
echo ' - Maximum stack depth exceeded';
break;
case JSON_ERROR_STATE_MISMATCH:
echo ' - Underflow or the modes mismatch';
break;
case JSON_ERROR_CTRL_CHAR:
echo ' - Unexpected control character found';
break;
case JSON_ERROR_SYNTAX:
echo ' - Syntax error, malformed JSON';
break;
case JSON_ERROR_UTF8:
echo ' - Malformed UTF-8 characters, possibly incorrectly encoded';
break;
default:
echo ' - Unknown error';
break;
}
if($json_object->message != ""){
error_log($web_result);
// Push to Matrix
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$web_hook_room);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode(array('secret' => $web_hook_secret, 'message' => "Error on the data warehouse")));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt( $ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json'));
$result = curl_exec($ch);
curl_close($ch);
exit();
}else{
foreach($json_object->data->transactions as $query) {
$result = mysqli_query ( $connect, $query->argument );
$records_inserted = $records_inserted +1;
}
// Push to Matrix
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$web_hook_room);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode(array('secret' => $web_hook_secret, 'message' => "Data warehouse inserted ".$records_inserted." queries")));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt( $ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json'));
$result = curl_exec($ch);
curl_close($ch);
}
}catch (Exception $e) {
$error_message = 'Exception Message: ' .$e->getMessage();
error_log($error_message);
error_log($web_result);
// Push to Matrix
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$web_hook_room);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode(array('secret' => $web_hook_secret, 'message' => "Error on the data warehouse")));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt( $ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json'));
$result = curl_exec($ch);
curl_close($ch);
}
}else{
echo '{ "message": "Invalid post variables", "data": {}}';
}
?>