SpasePapcev/cassandra-consistency-script
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

params to remove, udpated dirs

Browse source
This commit is contained in:
GbArc 2023-05-19 13:41:26 +02:00
parent 1f1bffb47e
commit bd68360866
1 changed files with 45 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

69
cassandra_consistency_script.php
View file

@ -12,6 +12,8 @@ class DataConsistencyChecker
const DEFAULT_PAGE_SIZE = 3; const DEFAULT_PAGE_SIZE = 3;
const CASSANDRA_USERNAME = 'cassandra'; const CASSANDRA_USERNAME = 'cassandra';
const CASSANDRA_PASSWORD = 'cassandra'; const CASSANDRA_PASSWORD = 'cassandra';
const CASSANDRA_KEYSPACE = 'tr_key';
private $_cluster; private $_cluster;
private $session; private $session;
private $cassandra; private $cassandra;
@ -19,7 +21,7 @@ class DataConsistencyChecker
private $directory; private $directory;
private $structured_directory = '/usr/share/nginx/html/testrail/db/cassandra'; private $structured_directory = '/usr/share/nginx/html/testrail/db/cassandra';
private $retrived_csv = '/usr/share/nginx/html/testrail/db/cassandra/result_from_physical_files.csv'; private $retrived_csv;
static $clientId; static $clientId;
private static $schemaVersion = 1; private static $schemaVersion = 1;
static $fileBucketCounter = 1; static $fileBucketCounter = 1;
@ -39,12 +41,16 @@ class DataConsistencyChecker
public function init(): void public function init(): void
{ {
$this->_cluster = Cassandra::cluster() $this->_cluster = Cassandra::cluster()
->withContactPoints('cassandra') ->withContactPoints(static::$cassandraHost)
->withPort(9042) ->withPort(9042)
->withCredentials(
static::CASSANDRA_USERNAME,
static::CASSANDRA_PASSWORD
)
->build(); ->build();
if ($this->_cluster) { if ($this->_cluster) {
try { try {
$this->cassandra = $this->_cluster->connect("tr_key"); $this->cassandra = $this->_cluster->connect(static::CASSANDRA_KEYSPACE);
} catch (Exception $e) { } catch (Exception $e) {
echo "err\n"; echo "err\n";
} }
@ -55,8 +61,8 @@ class DataConsistencyChecker
public function runFromCommandLine($arguments) public function runFromCommandLine($arguments)
{ {
$shortOptions = "hd:v"; $shortOptions = "hd:v:o:r:s:";
$longOptions = ["help", "directory:", "version:", "v"]; $longOptions = ["help", "directory:", "version:", "v", "output:", "o", "remove:", "r", "source:", "s"];
$options = getopt($shortOptions, $longOptions); $options = getopt($shortOptions, $longOptions);
@ -68,18 +74,32 @@ class DataConsistencyChecker
$directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null); $directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null);
$schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null); $schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null);
if ($directory === null || $schemaVersion === null) { $source = isset($options['source']) ? $options['source'] : (isset($options['s']) ? $options['s'] : null);
$remove = isset($options['remove']) ? $options['remove'] : (isset($options['r']) ? $options['r'] : null);
$this->structured_directory = isset($options['output']) ? $options['output'] : (isset($options['o']) ? $options['o'] : null);
if (($directory === null || $schemaVersion === null) && $remove === null && $source === null) {
echo "Missing Attachment directory or schema version.\n"; echo "Missing Attachment directory or schema version.\n";
exit; exit;
} }
if (!in_array($schemaVersion, [1, 2])) { if ($schemaVersion && !in_array($schemaVersion, [1, 2])) {
echo "Invalid schema version. Only versions 1 and 2 are supported.\n"; echo "Invalid schema version. Only versions 1 and 2 are supported.\n";
exit; exit;
} }
if ($this->structured_directory == null) {
$this->structured_directory = './';
}
static::$schemaVersion = (int)$schemaVersion; static::$schemaVersion = (int)$schemaVersion;
$this->directory = $directory;
$this->checkConsistency('attachment_file_info'); $this->retrived_csv = './result_from_physical_files.csv';
if ($remove && $source) {
$this->processAttachmentDeletionCSV($remove, $source);
}
else {
$this->checkConsistency('attachment_file_info');
}
exit; exit;
} }
@ -109,7 +129,7 @@ class DataConsistencyChecker
$dbEntries = $this->getDbEntries($tableName); $dbEntries = $this->getDbEntries($tableName);
$fileEntries = $this->getFileEntries($this->directory); $fileEntries = $this->getFileEntries($this->directory);
$this->process_files_in_directory($this->structured_directory); $this->process_files_in_directory($this->structured_directory);
$this->createCSVById($this->retrived_csv); //$this->createCSVById($this->retrived_csv);
} }
private function getFileEntries($directory) private function getFileEntries($directory)
@ -177,7 +197,7 @@ class DataConsistencyChecker
private function createPhysicalFileCSV($clientId, $entries) private function createPhysicalFileCSV($clientId, $entries)
{ {
$fileName = "physical_" . $clientId . ".csv"; $fileName = $this->structured_directory . "physical_" . $clientId . ".csv";
$csvFile = fopen($fileName, 'w'); $csvFile = fopen($fileName, 'w');
fputcsv($csvFile, ['id', 'size', 'creation_time']); fputcsv($csvFile, ['id', 'size', 'creation_time']);
foreach ($entries as $entry) { foreach ($entries as $entry) {
@ -194,7 +214,7 @@ class DataConsistencyChecker
private function createDBFileCSV($clientId, $entries) private function createDBFileCSV($clientId, $entries)
{ {
$fileName = "cassandra_" . (string) $clientId . ".csv"; $fileName = $this->structured_directory . "cassandra_" . (string) $clientId . ".csv";
$csvFile = fopen($fileName, 'w'); $csvFile = fopen($fileName, 'w');
$headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id']; $headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id'];
@ -305,7 +325,10 @@ class DataConsistencyChecker
$old_migrated_data = array_search('migrated_files_id', $headers); $old_migrated_data = array_search('migrated_files_id', $headers);
$query = "SELECT * FROM attachment_file_info WHERE id = ? ALLOW FILTERING"; $query = "SELECT * FROM attachment_file_info WHERE id = ? and client_id = ?";
if ($this->schema_version() == 2) {
$query = $query . ' and bucket = ?';
}
$statement = $this->cassandra->prepare($query); $statement = $this->cassandra->prepare($query);
foreach ($fileData as $row) { foreach ($fileData as $row) {
@ -492,7 +515,7 @@ class DataConsistencyChecker
if (!isset($physical_files_assoc[$id])) { if (!isset($physical_files_assoc[$id])) {
$missing_physical_files[] = [ $missing_physical_files[] = [
'id' => $id, 'id' => $id,
'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5], $data[6]], 'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5]],
]; ];
} }
} }
@ -527,7 +550,7 @@ class DataConsistencyChecker
} }
if (count($values) > 3) { if (count($values) > 3) {
$file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[0]]; $file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[6]];
} }
} }
@ -893,7 +916,7 @@ class DataConsistencyChecker
]; ];
if ($this->schema_version() == 1) { if ($this->schema_version() == 1) {
$query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? ALLOW FILTERING'); $query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ?');
} else { } else {
$q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\''; $q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\'';
echo "Q: [" . $q . "]\n"; echo "Q: [" . $q . "]\n";
@ -991,7 +1014,7 @@ class DataConsistencyChecker
$key = $this->_get_attachment_key($attachmentId); $key = $this->_get_attachment_key($attachmentId);
if ($key) { if ($key) {
if ($this->schema_version() === 1) { if ($this->schema_version() === 1) {
$query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ? ALLOW FILTERING'); $query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ?');
$arguments = [ $arguments = [
'client_id' => static::$clientId, 'client_id' => static::$clientId,
'id' => $key->id, 'id' => $key->id,
@ -1261,11 +1284,10 @@ class DataConsistencyChecker
$data->bucket = $values[8]; $data->bucket = $values[8];
$data->id = $values[9]; $data->id = $values[9];
} }
if ($data->source == $src) { if ($data->source === 'Attachment' && $src === 'cassandra') {
if ($src === 'cassandra') {
echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL; echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL;
fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL); fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL);
} else { } else if ($src === 'File' && $source === 'file'){
$filePath = $directory . $values[1]; $filePath = $directory . $values[1];
$thumb1Path = $directory . $values[1]; $thumb1Path = $directory . $values[1];
$thumb2Path = $directory . $values[1]; $thumb2Path = $directory . $values[1];
@ -1280,7 +1302,6 @@ class DataConsistencyChecker
echo "File not found: $filePath" . PHP_EOL; echo "File not found: $filePath" . PHP_EOL;
} }
} }
}
} }
} }