params to remove, udpated dirs
This commit is contained in:
GbArc
parent
1f1bffb47e
commit
bd68360866
1 changed files with 45 additions and 24 deletions
|
|
@ -12,6 +12,8 @@ class DataConsistencyChecker
|
|||
const DEFAULT_PAGE_SIZE = 3;
|
||||
const CASSANDRA_USERNAME = 'cassandra';
|
||||
const CASSANDRA_PASSWORD = 'cassandra';
|
||||
const CASSANDRA_KEYSPACE = 'tr_key';
|
||||
|
||||
private $_cluster;
|
||||
private $session;
|
||||
private $cassandra;
|
||||
|
|
@ -19,7 +21,7 @@ class DataConsistencyChecker
|
|||
private $directory;
|
||||
private $structured_directory = '/usr/share/nginx/html/testrail/db/cassandra';
|
||||
|
||||
private $retrived_csv = '/usr/share/nginx/html/testrail/db/cassandra/result_from_physical_files.csv';
|
||||
private $retrived_csv;
|
||||
static $clientId;
|
||||
private static $schemaVersion = 1;
|
||||
static $fileBucketCounter = 1;
|
||||
|
|
@ -39,12 +41,16 @@ class DataConsistencyChecker
|
|||
public function init(): void
|
||||
{
|
||||
$this->_cluster = Cassandra::cluster()
|
||||
->withContactPoints('cassandra')
|
||||
->withPort(9042)
|
||||
->withContactPoints(static::$cassandraHost)
|
||||
->withPort(9042)
|
||||
->withCredentials(
|
||||
static::CASSANDRA_USERNAME,
|
||||
static::CASSANDRA_PASSWORD
|
||||
)
|
||||
->build();
|
||||
if ($this->_cluster) {
|
||||
try {
|
||||
$this->cassandra = $this->_cluster->connect("tr_key");
|
||||
$this->cassandra = $this->_cluster->connect(static::CASSANDRA_KEYSPACE);
|
||||
} catch (Exception $e) {
|
||||
echo "err\n";
|
||||
}
|
||||
|
|
@ -55,8 +61,8 @@ class DataConsistencyChecker
|
|||
|
||||
public function runFromCommandLine($arguments)
|
||||
{
|
||||
$shortOptions = "hd:v";
|
||||
$longOptions = ["help", "directory:", "version:", "v"];
|
||||
$shortOptions = "hd:v:o:r:s:";
|
||||
$longOptions = ["help", "directory:", "version:", "v", "output:", "o", "remove:", "r", "source:", "s"];
|
||||
|
||||
$options = getopt($shortOptions, $longOptions);
|
||||
|
||||
|
|
@ -68,18 +74,32 @@ class DataConsistencyChecker
|
|||
$directory = isset($options['directory']) ? $options['directory'] : (isset($options['d']) ? $options['d'] : null);
|
||||
$schemaVersion = isset($options['version']) ? $options['version'] : (isset($options['v']) ? $options['v'] : null);
|
||||
|
||||
if ($directory === null || $schemaVersion === null) {
|
||||
$source = isset($options['source']) ? $options['source'] : (isset($options['s']) ? $options['s'] : null);
|
||||
$remove = isset($options['remove']) ? $options['remove'] : (isset($options['r']) ? $options['r'] : null);
|
||||
|
||||
$this->structured_directory = isset($options['output']) ? $options['output'] : (isset($options['o']) ? $options['o'] : null);
|
||||
|
||||
if (($directory === null || $schemaVersion === null) && $remove === null && $source === null) {
|
||||
echo "Missing Attachment directory or schema version.\n";
|
||||
exit;
|
||||
}
|
||||
if (!in_array($schemaVersion, [1, 2])) {
|
||||
if ($schemaVersion && !in_array($schemaVersion, [1, 2])) {
|
||||
echo "Invalid schema version. Only versions 1 and 2 are supported.\n";
|
||||
exit;
|
||||
}
|
||||
}
|
||||
if ($this->structured_directory == null) {
|
||||
$this->structured_directory = './';
|
||||
}
|
||||
|
||||
static::$schemaVersion = (int)$schemaVersion;
|
||||
|
||||
$this->checkConsistency('attachment_file_info');
|
||||
static::$schemaVersion = (int)$schemaVersion;
|
||||
$this->directory = $directory;
|
||||
$this->retrived_csv = './result_from_physical_files.csv';
|
||||
if ($remove && $source) {
|
||||
$this->processAttachmentDeletionCSV($remove, $source);
|
||||
}
|
||||
else {
|
||||
$this->checkConsistency('attachment_file_info');
|
||||
}
|
||||
exit;
|
||||
}
|
||||
|
||||
|
|
@ -109,7 +129,7 @@ class DataConsistencyChecker
|
|||
$dbEntries = $this->getDbEntries($tableName);
|
||||
$fileEntries = $this->getFileEntries($this->directory);
|
||||
$this->process_files_in_directory($this->structured_directory);
|
||||
$this->createCSVById($this->retrived_csv);
|
||||
//$this->createCSVById($this->retrived_csv);
|
||||
}
|
||||
|
||||
private function getFileEntries($directory)
|
||||
|
|
@ -177,7 +197,7 @@ class DataConsistencyChecker
|
|||
|
||||
private function createPhysicalFileCSV($clientId, $entries)
|
||||
{
|
||||
$fileName = "physical_" . $clientId . ".csv";
|
||||
$fileName = $this->structured_directory . "physical_" . $clientId . ".csv";
|
||||
$csvFile = fopen($fileName, 'w');
|
||||
fputcsv($csvFile, ['id', 'size', 'creation_time']);
|
||||
foreach ($entries as $entry) {
|
||||
|
|
@ -194,7 +214,7 @@ class DataConsistencyChecker
|
|||
|
||||
private function createDBFileCSV($clientId, $entries)
|
||||
{
|
||||
$fileName = "cassandra_" . (string) $clientId . ".csv";
|
||||
$fileName = $this->structured_directory . "cassandra_" . (string) $clientId . ".csv";
|
||||
$csvFile = fopen($fileName, 'w');
|
||||
|
||||
$headers = ['id', 'size', 'creation_time', 'filename', 'bucket', 'client_id', 'attachment_id'];
|
||||
|
|
@ -305,7 +325,10 @@ class DataConsistencyChecker
|
|||
$old_migrated_data = array_search('migrated_files_id', $headers);
|
||||
|
||||
|
||||
$query = "SELECT * FROM attachment_file_info WHERE id = ? ALLOW FILTERING";
|
||||
$query = "SELECT * FROM attachment_file_info WHERE id = ? and client_id = ?";
|
||||
if ($this->schema_version() == 2) {
|
||||
$query = $query . ' and bucket = ?';
|
||||
}
|
||||
$statement = $this->cassandra->prepare($query);
|
||||
|
||||
foreach ($fileData as $row) {
|
||||
|
|
@ -492,7 +515,7 @@ class DataConsistencyChecker
|
|||
if (!isset($physical_files_assoc[$id])) {
|
||||
$missing_physical_files[] = [
|
||||
'id' => $id,
|
||||
'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5], $data[6]],
|
||||
'file1' => [$id, $data[0], $data[1], $data[2], $data[3], $data[4], $data[5]],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -527,7 +550,7 @@ class DataConsistencyChecker
|
|||
}
|
||||
if (count($values) > 3) {
|
||||
|
||||
$file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[0]];
|
||||
$file_assoc[$values[0]] = [$values[1], $values[2], $values[3], $values[4], $values[5], $values[6]];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -893,7 +916,7 @@ class DataConsistencyChecker
|
|||
];
|
||||
|
||||
if ($this->schema_version() == 1) {
|
||||
$query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? ALLOW FILTERING');
|
||||
$query = $this->cassandra->prepare('SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ?');
|
||||
} else {
|
||||
$q = 'SELECT * FROM attachment_file_info WHERE id = ? AND client_id = ? AND bucket = \'' . $bucketId . '\'';
|
||||
echo "Q: [" . $q . "]\n";
|
||||
|
|
@ -991,7 +1014,7 @@ class DataConsistencyChecker
|
|||
$key = $this->_get_attachment_key($attachmentId);
|
||||
if ($key) {
|
||||
if ($this->schema_version() === 1) {
|
||||
$query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ? ALLOW FILTERING');
|
||||
$query = $this->cassandra->prepare('SELECT ' . $properties . ' FROM attachments WHERE id = ? AND client_id = ? AND project_id = ? AND entity_type = ?');
|
||||
$arguments = [
|
||||
'client_id' => static::$clientId,
|
||||
'id' => $key->id,
|
||||
|
|
@ -1261,11 +1284,10 @@ class DataConsistencyChecker
|
|||
$data->bucket = $values[8];
|
||||
$data->id = $values[9];
|
||||
}
|
||||
if ($data->source == $src) {
|
||||
if ($src === 'cassandra') {
|
||||
if ($data->source === 'Attachment' && $src === 'cassandra') {
|
||||
echo "will delete " . $data->clientId . " : " . $data->bucket . " : " . $data->id . PHP_EOL;
|
||||
fwrite($logHandle, "Deleted attachment: $data->id" . PHP_EOL);
|
||||
} else {
|
||||
} else if ($src === 'File' && $source === 'file'){
|
||||
$filePath = $directory . $values[1];
|
||||
$thumb1Path = $directory . $values[1];
|
||||
$thumb2Path = $directory . $values[1];
|
||||
|
|
@ -1280,7 +1302,6 @@ class DataConsistencyChecker
|
|||
echo "File not found: $filePath" . PHP_EOL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue