dave/Data_Warehouse
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Write to temp file

Browse Source
This commit is contained in:
Dave Umrysh 2021-05-07 15:54:46 +00:00
parent f57e9f79ea
commit 747ce7db81
1 changed files with 44 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
getDBLog.php
View File

@ -4,6 +4,11 @@ ini_set("memory_limit","8024M");
header('Access-Control-Allow-Origin: *'); header('Access-Control-Allow-Origin: *');
date_default_timezone_set('UTC'); date_default_timezone_set('UTC');
// if started from commandline, wrap parameters to $_POST and $_GET
if (!isset($_SERVER["HTTP_HOST"]) && isset($argv[1])) {
parse_str($argv[1], $_REQUEST);
}
$api_key = "cC7d2DHDnEwJTwiHLakhyjZGKLyWR"; $api_key = "cC7d2DHDnEwJTwiHLakhyjZGKLyWR";
if( isset($_REQUEST['apiKey']) && $_REQUEST['apiKey'] == $api_key && isset($_REQUEST['timestamp']) && is_numeric($_REQUEST['timestamp']) ) { if( isset($_REQUEST['apiKey']) && $_REQUEST['apiKey'] == $api_key && isset($_REQUEST['timestamp']) && is_numeric($_REQUEST['timestamp']) ) {
@ -13,22 +18,52 @@ if( isset($_REQUEST['apiKey']) && $_REQUEST['apiKey'] == $api_key && isset($_REQ
$output = '{ "message": "", "data": { "transactions": ['; $output = '{ "message": "", "data": { "transactions": [';
// Get the data from the current log $hour_of_day = date("G");
$query = 'select UNIX_TIMESTAMP(event_time) as event_time,argument from mysql.general_log where (upper(argument) like "%INSERT %" or upper(argument) like "%UPDATE %" or upper(argument) like "%CREATE %") and argument NOT LIKE "%mysql.rds_heartbeat2%" AND argument NOT LIKE "%INFORMATION_SCHEMA.GLOBAL_STATUS%"'; $chunk_size = 1000;
$result = mysqli_query ( $connect, $query); $starting = 0;
while ($thedata = mysqli_fetch_assoc($result)) { $still_have_rows = true;
$output = $output . '{"event_time" : "' . str_replace('"', '\"', $thedata["event_time"]) . '","argument" : "' . str_replace('"', '\"', str_replace('\"', '\\\"', str_replace("\'", "'", $thedata["argument"]))) . '"},'; $temp_path = "/mnt/cache/tmp/output.json";
}
$output = rtrim($output, ",");
$output = $output . ']}}';
$myfile = fopen($temp_path, "w") or die("Unable to open file!");
fwrite($myfile, $output);
// Get the data from the current log
$result = mysqli_query ( $connect, 'DROP TEMPORARY TABLE IF EXISTS logtemp.backupGeneralTable'.$hour_of_day);
$result = mysqli_query ( $connect, 'CREATE TEMPORARY TABLE logtemp.backupGeneralTable'.$hour_of_day.' AS (select UNIX_TIMESTAMP(event_time) as event_time,argument from mysql.general_log where (upper(argument) like "%INSERT %" or upper(argument) like "%UPDATE %" or upper(argument) like "%CREATE %") and argument NOT LIKE "%mysql.rds_heartbeat2%" AND argument NOT LIKE "%INFORMATION_SCHEMA.GLOBAL_STATUS%" AND argument NOT LIKE "%CREATE TEMPORARY TABLE%")');
// Rotate the general log table // Rotate the general log table
$query = 'CALL mysql.rds_rotate_general_log'; $query = 'CALL mysql.rds_rotate_general_log';
$result = mysqli_query ( $connect, $query); $result = mysqli_query ( $connect, $query);
$first_time = true;
while($still_have_rows){
$query = 'SELECT event_time,argument FROM logtemp.backupGeneralTable'.$hour_of_day.' LIMIT '.$starting.', '.$chunk_size;
$result = mysqli_query ( $connect, $query);
if(mysqli_num_rows($result)>0)
{
while ($thedata = mysqli_fetch_assoc($result)) {
if($first_time){
fwrite($myfile, '{"event_time" : "' . str_replace('"', '\"', $thedata["event_time"]) . '","argument" : "' . str_replace('"', '\"', str_replace('\"', '\\"', str_replace('\\', '\\\\', str_replace("\'", "'", str_replace("\t", " ", $thedata["argument"]))))) . '"}');
$first_time = false;
}else{
fwrite($myfile, ',{"event_time" : "' . str_replace('"', '\"', $thedata["event_time"]) . '","argument" : "' . str_replace('"', '\"', str_replace('\"', '\\"', str_replace('\\', '\\\\', str_replace("\'", "'", str_replace("\t", " ", $thedata["argument"]))))) . '"}');
}
}
$starting = $starting + $chunk_size;
}else{
$still_have_rows = false;
}
}
fwrite($myfile, ']}}');
fclose($myfile);
readfile($temp_path);
unlink($temp_path);
echo $output;
}else{ }else{
echo '{ "message": "Invalid post variables", "data": {}}'; echo '{ "message": "Invalid post variables", "data": {}}';
} }