diff --git a/cassandra_consistency_script.php b/cassandra_consistency_script.php index 68e0ef9..d7f253a 100644 --- a/cassandra_consistency_script.php +++ b/cassandra_consistency_script.php @@ -12,6 +12,8 @@ class DataConsistencyChecker const DEFAULT_PAGE_SIZE = 3; const CASSANDRA_USERNAME = 'cassandra'; const CASSANDRA_PASSWORD = 'cassandra'; + const CASSANDRA_KEYSPACE = 'tr_key'; + private $_cluster; private $session; private $cassandra; @@ -19,7 +21,7 @@ class DataConsistencyChecker private $directory; private $structured_directory = '/usr/share/nginx/html/testrail/db/cassandra'; - private $retrived_csv = '/usr/share/nginx/html/testrail/db/cassandra/result_from_physical_files.csv'; + private $retrived_csv; static $clientId; private static $schemaVersion = 1; static $fileBucketCounter = 1; @@ -39,12 +41,16 @@ class DataConsistencyChecker public function init(): void { $this->_cluster = Cassandra::cluster() - ->withContactPoints('cassandra') - ->withPort(9042) + ->withContactPoints(static::$cassandraHost) + ->withPort(9042) + ->withCredentials( + static::CASSANDRA_USERNAME, + static::CASSANDRA_PASSWORD + ) ->build(); if ($this->_cluster) { try { - $this->cassandra = $this->_cluster->connect("tr_key"); + $this->cassandra = $this->_cluster->connect(static::CASSANDRA_KEYSPACE); } catch (Exception $e) { echo "err\n"; } @@ -55,8 +61,8 @@ class DataConsistencyChecker public function runFromCommandLine($arguments) { - $shortOptions = "hd:v"; - $longOptions = ["help", "directory:", "version:", "v"]; + $shortOptions = "hd:v:o:r:s:"; + $longOptions = ["help", "directory:", "version:", "v", "output:", "o", "remove:", "r", "source:", "s"]; $options = getopt($shortOptions, $longOptions); @@ -68,18 +74,32 @@ class DataConsistencyChecker $directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null); $schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null); - if ($directory === null || $schemaVersion === null) { + $source = isset($options['source']) ? $options['source'] : (isset($options['s']) ? $options['s'] : null); + $remove = isset($options['remove']) ? $options['remove'] : (isset($options['r']) ? $options['r'] : null); + + $this->structured_directory = isset($options['output']) ? $options['output'] : (isset($options['o']) ? $options['o'] : null); + + if (($directory === null || $schemaVersion === null) && $remove === null && $source === null) { echo "Missing Attachment directory or schema version.\n"; exit; } - if (!in_array($schemaVersion, [1, 2])) { + if ($schemaVersion && !in_array($schemaVersion, [1, 2])) { echo "Invalid schema version. Only versions 1 and 2 are supported.\n"; exit; - } + } + if ($this->structured_directory == null) { + $this->structured_directory = './'; + } - static::$schemaVersion = (int)$schemaVersion; - - $this->checkConsistency('attachment_file_info'); + static::$schemaVersion = (int)$schemaVersion; + $this->directory = $directory; + $this->retrived_csv = './result_from_physical_files.csv'; + if ($remove && $source) { + $this->processAttachmentDeletionCSV($remove, $source); + } + else { + $this->checkConsistency('attachment_file_info'); + } exit; } @@ -109,7 +129,7 @@ class DataConsistencyChecker $dbEntries = $this->getDbEntries($tableName); $fileEntries = $this->getFileEntries($this->directory); $this->process_files_in_directory($this->structured_directory); - $this->createCSVById($this->retrived_csv); + //$this->createCSVById($this->retrived_csv); } private function getFileEntries($directory) @@ -177,7 +197,7 @@ class DataConsistencyChecker private function createPhysicalFileCSV($clientId, $entries) { - $fileName = "physical_" . $clientId . ".csv"; + $fileName = $this->structured_directory . "physical_" . $clientId . ".csv"; $csvFile = fopen($fileName, 'w'); fputcsv($csvFile, ['id', 'size', 'creation_time']); foreach ($entries as $entry) { @@ -194,7 +214,7 @@ class DataConsistencyChecker private function createDBFileCSV($clientId, $entries) { - $fileName = "cassandra_" . (string) $clientId . ".csv"; + $fileName = $this->structured_directory . "cassandra_" . (string) $clientId . ".csv"; $csvFile = fopen($fileName, 'w'); $headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id']; @@ -305,7 +325,10 @@ class DataConsistencyChecker $old_migrated_data = array_search('migrated_files_id', $headers); - $query = "SELECT * FROM attachment_file_info WHERE id = ? ALLOW FILTERING"; + $query = "SELECT * FROM attachment_file_info WHERE id = ? and client_id = ?"; + if ($this->schema_version() == 2) { + $query = $query . ' and bucket = ?'; + } $statement = $this->cassandra->prepare($query); foreach ($fileData as $row) { @@ -492,7 +515,7 @@ class DataConsistencyChecker if (!isset($physical_files_assoc[$id])) { $missing_physical_files[] = [ 'id' => $id, - 'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5], $data[6]], + 'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5]], ]; } } @@ -527,7 +550,7 @@ class DataConsistencyChecker } if (count($values) > 3) { - $file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[0]]; + $file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[6]]; } } @@ -893,7 +916,7 @@ class DataConsistencyChecker ]; if ($this->schema_version() == 1) { - $query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? ALLOW FILTERING'); + $query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ?'); } else { $q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\''; echo "Q: [" . $q . "]\n"; @@ -991,7 +1014,7 @@ class DataConsistencyChecker $key = $this->_get_attachment_key($attachmentId); if ($key) { if ($this->schema_version() === 1) { - $query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ? ALLOW FILTERING'); + $query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ?'); $arguments = [ 'client_id' => static::$clientId, 'id' => $key->id, @@ -1261,11 +1284,10 @@ class DataConsistencyChecker $data->bucket = $values[8]; $data->id = $values[9]; } - if ($data->source == $src) { - if ($src === 'cassandra') { + if ($data->source === 'Attachment' && $src === 'cassandra') { echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL; fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL); - } else { + } else if ($src === 'File' && $source === 'file'){ $filePath = $directory . $values[1]; $thumb1Path = $directory . $values[1]; $thumb2Path = $directory . $values[1]; @@ -1280,7 +1302,6 @@ class DataConsistencyChecker echo "File not found: $filePath" . PHP_EOL; } } - } } }