SpasePapcev/cassandra-consistency-script
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added additional changes and documentation

Browse source
This commit is contained in:
Spase 2023-05-19 19:01:57 +02:00
parent 1f1bffb47e
commit a35654a071
2 changed files with 357 additions and 305 deletions
Download patch file Download diff file Expand all files Collapse all files

439
cassandra_consistency_script.php
View file

@ -1,28 +1,17 @@
<?php <?php
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
use Cassandra\Cluster;
use Cassandra\ExecutionOptions;
use Cassandra\Session;
class DataConsistencyChecker class DataConsistencyChecker
{ {
const OLD_ATTACHMENTS_PREFIX = 'E_'; const DEFAULT_PAGE_SIZE = 30;
const DEFAULT_PAGE_SIZE = 3;
const CASSANDRA_USERNAME = 'cassandra';
const CASSANDRA_PASSWORD = 'cassandra';
private $_cluster; private $_cluster;
private $session; private $session;
private $cassandra; private $cassandra;
//private $directory = '/usr/share/nginx/html/testrail/attachments';
private $directory; private $directory;
private $structured_directory = '/usr/share/nginx/html/testrail/db/cassandra'; private $structured_directory;
private $retrived_csv;
private $retrived_csv = '/usr/share/nginx/html/testrail/db/cassandra/result_from_physical_files.csv';
static $clientId; static $clientId;
private static $schemaVersion = 1; private static $schemaVersion = 1;
static $fileBucketCounter = 1;
static $bucketMagic = 4; static $bucketMagic = 4;
static $cassandraHost = 'cassandra'; static $cassandraHost = 'cassandra';
@ -35,28 +24,43 @@ class DataConsistencyChecker
$this->runFromCommandLine($_SERVER['argv']); $this->runFromCommandLine($_SERVER['argv']);
} }
/**
* Initializes the Cassandra connection based on the configuration settings.
*
* @return void
*/
public function init(): void public function init(): void
{ {
$config = parse_ini_file("config.ini", true);
$this->_cluster = Cassandra::cluster() $this->_cluster = Cassandra::cluster()
->withContactPoints('cassandra') ->withContactPoints($config['CASSANDRA']['host'])
->withPort(9042) ->withPort(9042)
->withCredentials(
$config['CASSANDRA']['user'],
$config['CASSANDRA']['password']
)
->build(); ->build();
if ($this->_cluster) { if ($this->_cluster) {
try { try {
$this->cassandra = $this->_cluster->connect("tr_key"); $this->cassandra = $this->_cluster->connect($config['CASSANDRA']['keyspace']);
} catch (Exception $e) { } catch (Exception $e) {
echo "err\n"; echo "err\n";
} }
} }
// static::$bucketMagic = defined('CASSANDRA_BUCKET_MAGIC') ? (int) CASSANDRA_BUCKET_MAGIC : 4;
} }
/**
* Runs the script from the command line with the provided arguments.
*
* @param array $arguments The command line arguments.
* @return void
*/
public function runFromCommandLine($arguments) public function runFromCommandLine($arguments)
{ {
$shortOptions = "hd:v"; $shortOptions = "hd:v:o:r:s:";
$longOptions = ["help", "directory:", "version:", "v"]; $longOptions = ["help", "directory:", "version:", "v", "output:", "o", "remove:", "r", "source:", "s"];
$options = getopt($shortOptions, $longOptions); $options = getopt($shortOptions, $longOptions);
@ -68,22 +72,47 @@ class DataConsistencyChecker
$directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null); $directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null);
$schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null); $schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null);
if ($directory === null || $schemaVersion === null) { $source = isset($options['source']) ? $options['source'] : (isset($options['s']) ? $options['s'] : null);
$remove = isset($options['remove']) ? $options['remove'] : (isset($options['r']) ? $options['r'] : null);
$structured_directory = isset($options['output']) ? $options['output'] : (isset($options['o']) ? $options['o'] : null);
if (!file_exists($structured_directory)) {
mkdir($structured_directory, 0777, true);
}
$this->structured_directory = $structured_directory;
if (($directory === null || $schemaVersion === null) && $remove === null && $source === null) {
echo "Missing Attachment directory or schema version.\n"; echo "Missing Attachment directory or schema version.\n";
exit; exit;
} }
if (!in_array($schemaVersion, [1, 2])) { if ($schemaVersion && !in_array($schemaVersion, [1, 2])) {
echo "Invalid schema version. Only versions 1 and 2 are supported.\n"; echo "Invalid schema version. Only versions 1 and 2 are supported.\n";
exit; exit;
} }
if ($this->structured_directory == null) {
$this->structured_directory = './';
}
static::$schemaVersion = (int)$schemaVersion; static::$schemaVersion = (int) $schemaVersion;
$this->directory = $directory;
$this->retrived_csv = './result_from_physical_files.csv';
if ($remove && $source) {
$this->processAttachmentDeletionCSV($remove, $source);
} else {
$this->checkConsistency('attachment_file_info'); $this->checkConsistency('attachment_file_info');
if (is_dir($this->structured_directory)) {
$this->removeDirectory($this->structured_directory);
}
}
exit; exit;
} }
private function displayHelpMessage() /**
* Displays the help message with instructions on how to use the script.
*
* @return void
*/
private function displayHelpMessage(): void
{ {
$helpMessage = <<<EOT $helpMessage = <<<EOT
Usage: Usage:
@ -93,26 +122,67 @@ class DataConsistencyChecker
-h, --help Display this help screen. -h, --help Display this help screen.
--version Set the schema version. --version Set the schema version.
--directory Set the directory path for attachments. --directory Set the directory path for attachments.
--output Set the folder for temp files
--remove We need to choose between file and cassandra what we want to remove
Example: Example:
php script.php --directory /path/to/directory --version schema_version php script_name --version schema_version --directory=/path/to/directory --output ./out/
For Delete:
php script_name --remove result_from_cassandra_entries.csv --source file - to remove missing physical files
php script_name --remove result_from_physical_files.csv --source cassandra - to remove missing cassandra entries
EOT; EOT;
echo $helpMessage; echo $helpMessage;
} }
/**
* Recursively removes a directory and its contents.
*
* @param string $directory The directory path to be removed.
* @return void
*/
private function removeDirectory($directory): void
{
if (!is_dir($directory)) {
return;
}
$files = array_diff(scandir($directory), ['.', '..']);
foreach ($files as $file) {
$path = $directory . '/' . $file;
if (is_dir($path)) {
$this->removeDirectory($path);
} else {
unlink($path);
}
}
rmdir($directory);
}
/**
* Checks the consistency between database entries and file entries.
*
* @param string $tableName The name of the table in the database to check consistency for.
* @return void
*/
public function checkConsistency($tableName) public function checkConsistency($tableName)
{ {
//initialize the cassandra connection
$this->init(); $this->init();
$dbEntries = $this->getDbEntries($tableName); $dbEntries = $this->getDbEntries($tableName);
$fileEntries = $this->getFileEntries($this->directory); $fileEntries = $this->getFileEntries($this->directory);
$this->process_files_in_directory($this->structured_directory); $this->process_files_in_directory($this->structured_directory);
$this->createCSVById($this->retrived_csv);
} }
private function getFileEntries($directory) /**
* Retrieves file entries from a directory and organizes them based on dynamic values.
*
* @param string $directory The directory path to retrieve file entries from.
* @return array An array containing the file entries organized by dynamic values.
*/
private function getFileEntries($directory): array
{ {
$files = glob($directory . '/*'); $files = glob($directory . '/*');
$entries = []; $entries = [];
@ -120,15 +190,11 @@ class DataConsistencyChecker
foreach ($files as $file) { foreach ($files as $file) {
if (is_file($file)) { if (is_file($file)) {
$fileName = basename($file); $fileName = basename($file);
// Skip files with specific suffixes
if (strpos($fileName, '-thumb1') !== false || strpos($fileName, '-thumb2') !== false) { if (strpos($fileName, '-thumb1') !== false || strpos($fileName, '-thumb2') !== false) {
continue; continue;
} }
$dashParts = explode("-", $fileName, 2); $dashParts = explode("-", $fileName, 2);
$dotParts = explode(".", $fileName); $dotParts = explode(".", $fileName);
if (count($dashParts) === 2) { if (count($dashParts) === 2) {
$clientId = $dashParts[0]; $clientId = $dashParts[0];
$id = $dashParts[1]; $id = $dashParts[1];
@ -140,16 +206,12 @@ class DataConsistencyChecker
$fileParts = $dotParts; $fileParts = $dotParts;
$delimiter = '.'; $delimiter = '.';
} else { } else {
// Handle cases where the file name does not contain either a dash or a dot
continue; continue;
} }
$filePath = $file; $filePath = $file;
$size = filesize($filePath); $size = filesize($filePath);
// Use file modification time (filemtime()) instead of file creation time (filectime())
$creationTime = date('Y-m-d H:i:s', filemtime($filePath)); $creationTime = date('Y-m-d H:i:s', filemtime($filePath));
$dynamicValue = substr($id, 0, 2); $dynamicValue = substr($id, 0, 2);
$creationTime = str_replace('"', '', $creationTime); $creationTime = str_replace('"', '', $creationTime);
@ -162,7 +224,6 @@ class DataConsistencyChecker
]; ];
} }
} }
// create CSV file for each client ID's physical file entries // create CSV file for each client ID's physical file entries
foreach ($entries as $clientId => $clientEntries) { foreach ($entries as $clientId => $clientEntries) {
$this->createPhysicalFileCSV($clientId, $clientEntries); $this->createPhysicalFileCSV($clientId, $clientEntries);
@ -172,12 +233,17 @@ class DataConsistencyChecker
} }
/**
* Creates a CSV file containing the physical file entries for a specific client ID.
*
* @param string $clientId The client ID.
* @param array $entries An array containing the physical file entries for the client.
* @return void
*/
private function createPhysicalFileCSV($clientId, $entries) private function createPhysicalFileCSV($clientId, $entries)
{ {
$fileName = "physical_" . $clientId . ".csv"; $fileName = $this->structured_directory . "physical_" . $clientId . ".csv";
$csvFile = fopen($fileName, 'w'); $csvFile = fopen($fileName, 'w');
fputcsv($csvFile, ['id', 'size', 'creation_time']); fputcsv($csvFile, ['id', 'size', 'creation_time']);
foreach ($entries as $entry) { foreach ($entries as $entry) {
@ -191,10 +257,17 @@ class DataConsistencyChecker
} }
/**
* Creates a CSV file containing the entries for a specific client ID.
*
* @param string $clientId The client ID.
* @param array $entries An array containing the entries for the client.
* @return void
*/
private function createDBFileCSV($clientId, $entries) private function createDBFileCSV($clientId, $entries): void
{ {
$fileName = "cassandra_" . (string) $clientId . ".csv"; $fileName = $this->structured_directory . "cassandra_" . (string) $clientId . ".csv";
$csvFile = fopen($fileName, 'w'); $csvFile = fopen($fileName, 'w');
$headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id']; $headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id'];
@ -217,12 +290,18 @@ class DataConsistencyChecker
fclose($csvFile); fclose($csvFile);
} }
/**
* Retrieves entries from a database table.
*
* @param string $tableName The name of the database table.
* @return array An array containing the retrieved entries.
*/
private function getDbEntries($tableName) private function getDbEntries($tableName)
{ {
if($this->schema_version() === 1) { if ($this->schema_version() === 1) {
$query = "SELECT client_id, id, size, filename, created_on FROM $tableName"; $query = "SELECT client_id, id, size, filename, created_on FROM $tableName";
} else { } else {
$query = "SELECT client_id, id, size, filename, created_on, bucket FROM $tableName"; $query = "SELECT client_id, id, size, filename, created_on, bucket FROM $tableName";
@ -238,6 +317,7 @@ class DataConsistencyChecker
$entries = []; $entries = [];
while ($result) { while ($result) {
foreach ($result as $row) { foreach ($result as $row) {
if (preg_match('/^[0-9]+$/', $row['id'])) { if (preg_match('/^[0-9]+$/', $row['id'])) {
$dotParts = explode(".", $row['filename'], 2); $dotParts = explode(".", $row['filename'], 2);
$dynamicValue = substr($dotParts[1], 0, 2); $dynamicValue = substr($dotParts[1], 0, 2);
@ -245,7 +325,6 @@ class DataConsistencyChecker
$date = date('Y-m-d H:i:s', $timestamp); $date = date('Y-m-d H:i:s', $timestamp);
$creationTime = str_replace('"', '', $date); $creationTime = str_replace('"', '', $date);
$entry = [ $entry = [
"id" => $row['id'], "id" => $row['id'],
"size" => (string) $row['size'], "size" => (string) $row['size'],
"creation_time" => $creationTime, "creation_time" => $creationTime,
@ -292,81 +371,16 @@ class DataConsistencyChecker
return $entries; return $entries;
} }
private function createCSVById($csvFile)
{
$data = [];
/**
* Compares a file entry with its corresponding entry in the Cassandra file association.
*
* @param string $id The ID of the file entry.
* @param array $data An array containing data of the file entry.
* @param array $cassandra_file_assoc The Cassandra file association.
* @return array|null An array containing mismatched entries, or null if the entries match.
*/
$fileData = array_map('str_getcsv', file($csvFile));
$headers = array_shift($fileData);
$idIndex = array_search('File path', $headers);
$old_migrated_data = array_search('migrated_files_id', $headers);
$query = "SELECT * FROM attachment_file_info WHERE id = ? ALLOW FILTERING";
$statement = $this->cassandra->prepare($query);
foreach ($fileData as $row) {
if ($old_migrated_data !== false && isset($row[$old_migrated_data])) {
$id = $row[$old_migrated_data];
} else {
$id = $row[2];
}
$options = ['arguments' => [$id]];
$result = $this->cassandra->execute($statement, $options);
foreach ($result as $row) {
$data[] = $row;
}
}
$fileName = "data_by_id.csv";
$csvFile = fopen($fileName, 'w');
fputcsv($csvFile, $headers);
foreach ($data as $entry) {
$rowData = [];
foreach ($headers as $field) {
$value = isset($entry[$field]) ? $entry[$field] : '';
$rowData[] = $value;
}
fputcsv($csvFile, $rowData);
}
fclose($csvFile);
}
private function parseCSVFile($file)
{
$file_contents = file_get_contents($file);
$file_lines = explode("\n", $file_contents);
$file_assoc = array();
foreach ($file_lines as $line) {
if ($line === reset($file_lines)) {
continue;
}
$values = explode(",", $line);
if (count($values) == 3) {
$values[2] = str_replace('"', '', $values[2]);
$file_assoc[$values[0]] = [$values[1], $values[2]];
}
}
return $file_assoc;
}
// Helper function to read file contents and parse CSV lines
private function compareFileEntries($id, $data, $cassandra_file_assoc) private function compareFileEntries($id, $data, $cassandra_file_assoc)
{ {
if (!isset($cassandra_file_assoc[$id])) { if (!isset($cassandra_file_assoc[$id])) {
@ -391,13 +405,27 @@ class DataConsistencyChecker
return null; return null;
} }
private function getFileLines($file) /**
* Retrieves the lines of a file and returns them as an array.
*
* @param string $file The path to the file.
* @return array An array containing the lines of the file.
*/
private function getFileLines($file): array
{ {
$file_contents = file_get_contents($file); $file_contents = file_get_contents($file);
return explode("\n", $file_contents); return explode("\n", $file_contents);
} }
private function filterAndMapEntries($entries) /**
* Filters out null entries and maps the remaining entries to their first element.
*
* @param array $entries An array containing entries to be filtered and mapped.
* @return array An array of filtered and mapped entries.
*/
private function filterAndMapEntries($entries): array
{ {
$filtered_entries = array_filter($entries); $filtered_entries = array_filter($entries);
$mapped_entries = array_map(function ($entry) { $mapped_entries = array_map(function ($entry) {
@ -405,7 +433,14 @@ class DataConsistencyChecker
}, $filtered_entries); }, $filtered_entries);
return array_values($mapped_entries); return array_values($mapped_entries);
} }
private function process_files_in_directory($dir)
/**
* Processes files in a directory, performs comparisons, and generates CSV and HTML reports.
*
* @param string $dir The directory path containing the files to be processed.
* @return void
*/
private function process_files_in_directory($dir): void
{ {
$files = glob($dir . '/*.csv'); $files = glob($dir . '/*.csv');
$physical_files = array(); $physical_files = array();
@ -436,8 +471,6 @@ class DataConsistencyChecker
$compared_physical[] = $this->compare_csv_files($physical_file, $cassandra_file); $compared_physical[] = $this->compare_csv_files($physical_file, $cassandra_file);
$compared_cassandra[] = $this->compare_csv_files($cassandra_file, $physical_file); $compared_cassandra[] = $this->compare_csv_files($cassandra_file, $physical_file);
$physical_entries = $this->filterAndMapEntries($compared_cassandra); $physical_entries = $this->filterAndMapEntries($compared_cassandra);
//echo "phe: " . var_dump($physical_entries) . PHP_EOL;
$cassandra_entries = $this->filterAndMapEntries($compared_physical); $cassandra_entries = $this->filterAndMapEntries($compared_physical);
} else { } else {
@ -484,33 +517,35 @@ class DataConsistencyChecker
if (!file_exists($physical_file)) { if (!file_exists($physical_file)) {
$cassandra_file_lines = $this->getFileLines($cassandra_file); $cassandra_file_lines = $this->getFileLines($cassandra_file);
// echo "cfl: " . var_dump($cassandra_file_lines) . PHP_EOL;
$file_assoc = $this->buildFileAssociation($cassandra_file_lines); $file_assoc = $this->buildFileAssociation($cassandra_file_lines);
foreach ($file_assoc as $id => $data) { foreach ($file_assoc as $id => $data) {
if (!isset($physical_files_assoc[$id])) { if (!isset($physical_files_assoc[$id])) {
$missing_physical_files[] = [ $missing_physical_files[] = [
'id' => $id, 'id' => $id,
'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5], $data[6]], 'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5]],
]; ];
} }
} }
} }
} }
$result_from_cassandra_entries = array_unique(array_merge($cassandra_entries, $missing_cassandra_entries), SORT_REGULAR); $result_from_cassandra_entries = array_unique(array_merge($cassandra_entries, $missing_cassandra_entries), SORT_REGULAR);
// var_dump($missing_physical_files);
$result_from_physical_files = array_unique(array_merge($physical_entries, $missing_physical_files), SORT_REGULAR); $result_from_physical_files = array_unique(array_merge($physical_entries, $missing_physical_files), SORT_REGULAR);
$this->generateCsvReportForDbEntries($result_from_cassandra_entries, 'result_from_cassandra_entries.csv'); $this->generateCsvReportForDbEntries($result_from_cassandra_entries, 'result_from_cassandra_entries.csv');
$this->generateHtmlReport($result_from_cassandra_entries, 'cassandra.html'); $this->generateHtmlReport($result_from_cassandra_entries, 'cassandra.html');
$this->generateCsvReportForPhysicalFiles($result_from_physical_files, 'result_from_physical_files.csv'); $this->generateCsvReportForPhysicalFiles($result_from_physical_files, 'result_from_physical_files.csv');
$this->generateHtmlReport($result_from_physical_files, 'physical.html'); $this->generateHtmlReport($result_from_physical_files, 'physical.html');
// $this->compareCSVFilesTransform($this->structured_directory . 'cassandra_all_entries.csv', $this->structured_directory . 'result_from_cassandra_entries.csv', $this->structured_directory . 'final_file.csv');
// $this->deletePhysicalFilesFromCsv('result_from_cassandra_entries.csv');
} }
private function buildFileAssociation($file_lines) /**
* Builds an associative array from file lines.
*
* @param array $file_lines An array containing lines of a file.
* @return array An associative array representing the file association.
*/
private function buildFileAssociation($file_lines): array
{ {
$file_assoc = []; $file_assoc = [];
@ -520,22 +555,28 @@ class DataConsistencyChecker
} }
$values = explode(",", $line); $values = explode(",", $line);
$values[2] = str_replace('"', '', $values[2]);
if (count($values) == 3) { if (count($values) == 3) {
$file_assoc[$values[0]] = [$values[1], $values[2]]; $file_assoc[$values[0]] = [$values[1], $values[2]];
} }
if (count($values) > 3) { if (count($values) > 3) {
$file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[0]]; $file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[6]];
} }
} }
return $file_assoc; return $file_assoc;
} }
/**
* Compares two CSV files and returns missing entries or entries with mismatched data.
*
* @param string $file1_path The file path of the first CSV file.
* @param string $file2_path The file path of the second CSV file.
* @return array An array containing missing entries or entries with mismatched data.
*/
private function compare_csv_files($file1_path, $file2_path) private function compare_csv_files($file1_path, $file2_path): array
{ {
$file1_data = array_map('str_getcsv', file($file1_path)); $file1_data = array_map('str_getcsv', file($file1_path));
$file2_data = array_map('str_getcsv', file($file2_path)); $file2_data = array_map('str_getcsv', file($file2_path));
@ -594,7 +635,7 @@ class DataConsistencyChecker
]; ];
} else { } else {
$file2_data = $file2_assoc[$id]['file2']; $file2_data = $file2_assoc[$id]['file2'];
if ($data['file1'][1] !== $file2_data[1] || $data['file1'][2] !== $file2_data[2]) { if ($data['file1'][0] !== $file2_data[0] || $data['file1'][1] !== $file2_data[1]) {
$missing_entries[] = [ $missing_entries[] = [
'id' => $id, 'id' => $id,
'file1' => $data['file1'], 'file1' => $data['file1'],
@ -619,16 +660,13 @@ class DataConsistencyChecker
private function generateCsvReportForPhysicalFiles(array $inconsistentFiles, string $name): void private function generateCsvReportForPhysicalFiles(array $inconsistentFiles, string $name): void
{ {
$fp = fopen($name, 'w'); $fp = fopen($name, 'w');
fputcsv($fp, ['File/Attachment', 'File path', 'File name', 'Thumb 1', 'Thumb 2', 'Size', 'Creation Time', 'ClientId', 'Bucket' , 'Id']); fputcsv($fp, ['File/Attachment', 'File path', 'File name', 'Thumb 1', 'Thumb 2', 'Size', 'Creation Time', 'ClientId', 'Bucket', 'Id']);
foreach ($inconsistentFiles as $row) { foreach ($inconsistentFiles as $row) {
$file1Value = $row['file1'][0];
$check_value = $row['id']; $check_value = $row['id'];
//$check_value = is_numeric($row['id']) ? $row['file1'][3] : $file1Value; $filePath = $check_value ? $check_value : $row['file2'][0];
$filePath = $this->directory . '/' . $check_value ? $this->directory . '/' . $check_value : $this->directory . '/' . $row['file2'][0];
if (is_numeric($row['id'])) { if (is_numeric($row['id'])) {
$filePath = $this->directory . '/' . $row['file1'][3]; $filePath = $row['file1'][3];
$check_value = $row['file1'][3]; $check_value = $row['file1'][3];
} }
@ -658,7 +696,13 @@ class DataConsistencyChecker
chmod($name, 0666); chmod($name, 0666);
} }
/**
* Generates a CSV report for inconsistent database entries.
*
* @param array $inconsistentFiles An array containing inconsistent file data.
* @param string $name The name of the CSV report file to be generated.
* @return void
*/
private function generateCsvReportForDbEntries(array $inconsistentFiles, string $name): void private function generateCsvReportForDbEntries(array $inconsistentFiles, string $name): void
{ {
@ -666,14 +710,18 @@ class DataConsistencyChecker
fputcsv($fp, ['File/Attachment', 'Entry Path', 'Entry Name', 'Thumb 1', 'Thumb 2', 'Size', 'Creation Time']); fputcsv($fp, ['File/Attachment', 'Entry Path', 'Entry Name', 'Thumb 1', 'Thumb 2', 'Size', 'Creation Time']);
foreach ($inconsistentFiles as $row) { foreach ($inconsistentFiles as $row) {
$filePath = $this->directory . '/' . $row['file1'][0] ? $this->directory . '/' . $row['file1'][0] : $this->directory . '/' . $row['file2'][0];
$filePath = $row['file1'][0] ? $row['file1'][0] : $row['file2'][0];
$size = isset($row['file1'][1]) ? (string) $row['file1'][1] : filesize($filePath); $size = isset($row['file1'][1]) ? (string) $row['file1'][1] : filesize($filePath);
$creationTime = isset($row['file1'][2]) ? (string) $row['file1'][2] : date('Y-m-d H:i:s', filectime($filePath)); $creationTime = isset($row['file1'][2]) ? (string) $row['file1'][2] : date('Y-m-d H:i:s', filectime($filePath));
$thumb1 = isset($row['thumb1']) ? $row['thumb1'] : ''; $thumb1 = $row['file1'][0] . '-thumb1';
$thumb2 = isset($row['thumb2']) ? $row['thumb2'] : ''; $thumb2 = $row['file1'][0] . '-thumb2';
if (is_string($row['id']) && strpos($row['id'], '.') !== false) {
$old_attachment = explode('.', $row['id'])[0];
$thumb1 = '';
$thumb2 = '';
}
fputcsv($fp, [ fputcsv($fp, [
'Attachment', 'File',
$filePath, $filePath,
$row['file1'][0], $row['file1'][0],
$thumb1, $thumb1,
@ -696,6 +744,7 @@ class DataConsistencyChecker
*/ */
private function generateHtmlReport(array $inconsistentFiles, string $name): void private function generateHtmlReport(array $inconsistentFiles, string $name): void
{ {
$file = fopen($name, 'w'); $file = fopen($name, 'w');
if (!$file) { if (!$file) {
throw new Exception('Failed to open the file for writing.'); throw new Exception('Failed to open the file for writing.');
@ -734,7 +783,16 @@ class DataConsistencyChecker
fclose($file); fclose($file);
} }
function compareCSVFilesTransform($firstFile, $secondFile, $finalFile) /**
* Compares two CSV files and creates a new CSV file containing the matching entries.
*
* @param string $firstFile The path to the first CSV file.
* @param string $secondFile The path to the second CSV file.
* @param string $finalFile The path to the final CSV file to be created.
* @return void
*/
function compareCSVFilesTransform($firstFile, $secondFile, $finalFile): void
{ {
// Read the first CSV file // Read the first CSV file
$firstData = array_map('str_getcsv', file($firstFile)); $firstData = array_map('str_getcsv', file($firstFile));
@ -790,6 +848,12 @@ class DataConsistencyChecker
fclose($finalCsvFile); fclose($finalCsvFile);
} }
/**
* Deletes physical files based on the entries listed in a CSV file.
*
* @param string $csvFile The path to the CSV file containing the list of files to delete.
* @return void
*/
private function deletePhysicalFilesFromCsv(string $csvFile): void private function deletePhysicalFilesFromCsv(string $csvFile): void
{ {
@ -842,28 +906,6 @@ class DataConsistencyChecker
} }
private static function _is_uuid(string $id): bool
{
$regex = '/^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}$/';
return preg_match($regex, $id);
}
/**
* Check if the attachment string is a valid migrated attachment
*
* @param string $attachmentId
*
* @return bool
*/
private static function _isValidMigratedAttachment(string $attachmentId = ''): bool
{
return str::sub($attachmentId, 0, 2) === OLD_ATTACHMENTS_PREFIX;
}
/* ATTACHMENT DELETION ===================================================================================================================== */
/** /**
* Returns cassandra schema version * Returns cassandra schema version
* *
@ -893,7 +935,7 @@ class DataConsistencyChecker
]; ];
if ($this->schema_version() == 1) { if ($this->schema_version() == 1) {
$query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? ALLOW FILTERING'); $query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ?');
} else { } else {
$q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\''; $q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\'';
echo "Q: [" . $q . "]\n"; echo "Q: [" . $q . "]\n";
@ -956,7 +998,6 @@ class DataConsistencyChecker
*/ */
private function _get_attachment_key(string $id): ?object private function _get_attachment_key(string $id): ?object
{ {
//echo "GET KEY!\n";
$result = null; $result = null;
$query = $this->cassandra->prepare('SELECT * from attachment_ids where id = ? AND bucket = ? AND client_id = ?'); $query = $this->cassandra->prepare('SELECT * from attachment_ids where id = ? AND bucket = ? AND client_id = ?');
$arguments = [ $arguments = [
@ -964,14 +1005,11 @@ class DataConsistencyChecker
'bucket' => $this->_set_bucket($id), 'bucket' => $this->_set_bucket($id),
'id' => $id, 'id' => $id,
]; ];
//echo "args: " . var_dump($arguments, true) . PHP_EOL;
$data = $this->cassandra->execute($query, ['arguments' => $arguments]); $data = $this->cassandra->execute($query, ['arguments' => $arguments]);
echo "GK2!!!!";
//var_dump($data);
if ($data && $data->valid()) { if ($data && $data->valid()) {
$result = (object) $data->current(); $result = (object) $data->current();
} }
//var_dump($result); ;
return $result; return $result;
} }
@ -987,11 +1025,11 @@ class DataConsistencyChecker
$attachment = null; $attachment = null;
try { try {
$properties = '*'; $properties = '*';
echo "attid: ". $attachmentId . PHP_EOL; echo "attid: " . $attachmentId . PHP_EOL;
$key = $this->_get_attachment_key($attachmentId); $key = $this->_get_attachment_key($attachmentId);
if ($key) { if ($key) {
if ($this->schema_version() === 1) { if ($this->schema_version() === 1) {
$query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ? ALLOW FILTERING'); $query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ?');
$arguments = [ $arguments = [
'client_id' => static::$clientId, 'client_id' => static::$clientId,
'id' => $key->id, 'id' => $key->id,
@ -1085,7 +1123,6 @@ class DataConsistencyChecker
'id' => $id, 'id' => $id,
]; ];
$fileInfo = $this->get_info($clientId, $bucketId, $id); $fileInfo = $this->get_info($clientId, $bucketId, $id);
//var_dump($fileInfo);
if ($fileInfo) { if ($fileInfo) {
$this->_update_attachment_stats(false, $fileInfo->size); $this->_update_attachment_stats(false, $fileInfo->size);
} else { } else {
@ -1221,7 +1258,7 @@ class DataConsistencyChecker
* file,/tmp/testx3f,testx3f,testx3f-thumb1,testx3f-thumb2,4343,20-02-22 13:30,,, * file,/tmp/testx3f,testx3f,testx3f-thumb1,testx3f-thumb2,4343,20-02-22 13:30,,,
* cassandra,/test1/testx3,testx3,testx3-thumb1,testx3-thumb2,4343,20-02-22 13:30,1,3,3abc-def * cassandra,/test1/testx3,testx3,testx3-thumb1,testx3-thumb2,4343,20-02-22 13:30,1,3,3abc-def
*/ */
public function processAttachmentDeletionCSV(string $file, string $src) : void public function processAttachmentDeletionCSV(string $file, string $src): void
{ {
echo "Before proceeding with the deletion, make sure you have a backup of your data." . PHP_EOL; echo "Before proceeding with the deletion, make sure you have a backup of your data." . PHP_EOL;
@ -1236,7 +1273,6 @@ class DataConsistencyChecker
$file_lines = explode("\n", $file_contents); $file_lines = explode("\n", $file_contents);
$logFile = 'deleted_files.log'; $logFile = 'deleted_files.log';
$logHandle = fopen($logFile, 'a'); $logHandle = fopen($logFile, 'a');
$directory = $this->directory . '/';
foreach ($file_lines as $line) { foreach ($file_lines as $line) {
if ($line === reset($file_lines)) { if ($line === reset($file_lines)) {
@ -1256,23 +1292,30 @@ class DataConsistencyChecker
$data->thumb2 = $values[4]; $data->thumb2 = $values[4];
$data->size = $values[5]; $data->size = $values[5];
$data->created = $values[6]; $data->created = $values[6];
if(count($values) >= 10) { $path = dirname($data->path);
if (count($values) >= 10) {
$data->clientId = (int) $values[7]; $data->clientId = (int) $values[7];
$data->bucket = $values[8]; $data->bucket = $values[8];
$data->id = $values[9]; $data->id = $values[9];
} }
if ($data->source == $src) { if ($data->source === 'Attachment' && $src === 'cassandra') {
if ($src === 'cassandra') {
echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL; echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL;
$this->deleteAttachment($data->clientId, $data->bucket, $data->id);
fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL); fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL);
} else { } else if ($data->source === 'File' && $src === 'file') {
$filePath = $directory . $values[1]; $filePath = $values[1];
$thumb1Path = $directory . $values[1]; $thumb1Path = $path . '/' . $values[3];
$thumb2Path = $directory . $values[1]; $thumb2Path = $path . '/' . $values[4];
if (file_exists($thumb1Path) && file_exists($thumb2Path)) {
files::delete($thumb1Path);
files::delete($thumb2Path);
fwrite($logHandle, "Deleted thumbnail: " . $thumb1Path . PHP_EOL);
fwrite($logHandle, "Deleted thumbnail: " . $thumb2Path . PHP_EOL);
}
if (file_exists($filePath)) { if (file_exists($filePath)) {
// unlink($filePath); unlink($filePath);
// unlink($thumb1Path); unlink($thumb1Path);
// unlink($thumb2Path); unlink($thumb2Path);
echo "File deleted: $filePath" . PHP_EOL; echo "File deleted: $filePath" . PHP_EOL;
// Write the deleted file path to the log file // Write the deleted file path to the log file
fwrite($logHandle, "Deleted file: $filePath" . PHP_EOL); fwrite($logHandle, "Deleted file: $filePath" . PHP_EOL);
@ -1282,7 +1325,6 @@ class DataConsistencyChecker
} }
} }
} }
}
fclose($logHandle); fclose($logHandle);
@ -1290,10 +1332,5 @@ class DataConsistencyChecker
} }
} }
#$options = getopt('', ['directory:']);
$checker = new DataConsistencyChecker(); $checker = new DataConsistencyChecker();
$checker->checkConsistency("attachment_file_info", true); $checker->checkConsistency("attachment_file_info", true);
#$checker::$cassandraHost = 'localhost';
#$checker->init($options);
#$checker->deleteAttachment("1", "f", "ff29ead0-8696-4ef1-8120-538d6dd7efd1");
#$checker->processAttachmentDeletionCSV("todelete.csv", "file");

15
config.ini Normal file
View file

@ -0,0 +1,15 @@
[SQL]
driver = mysql
host = 127.0.0.1
databaseName = testrail
user = testrail
password = 123456789
port = 6666
[CASSANDRA]
host = cassandra
databaseName = testrail
user = casandra
password = cassandra
port = 9042
keyspace = testrail