gramps
373ebc8c93
952 days continuous production uptime, 40k+ tp/s single node. Original corpo Bitbucket history not included — clean archive commit.
358 lines
19 KiB
PHP
358 lines
19 KiB
PHP
<?php /** @noinspection PhpComposerExtensionStubsInspection */
|
|
/**
|
|
* mongoConfig -- utility script
|
|
*
|
|
* mongoConfig is a utility script that updates the existing indexes in all mongo collections in the database.
|
|
*
|
|
* this program should be run whenever new templates are added to the source tree, or when an existing template
|
|
* is edited. Actually, the program should just be run whenever the source is pushed to a new environment.
|
|
*
|
|
* the script reads all of the files in the ./classes/templates directory and every template file is parsed. If
|
|
* the template has been configured for a mongo schema, then it's indexing information is scraped.
|
|
*
|
|
* Processing generally follows the formula of:
|
|
*
|
|
* -- process all the compound indexes first
|
|
* -- process all the other indexes individually
|
|
* -- process individual index options (unique, sparse, ttl)
|
|
*
|
|
* NOTES:
|
|
* ------
|
|
* this program writes it's output to ./logs/mongoConfig.log
|
|
* The createIndex() method only creates an index if an index of the same specification does not already exist.
|
|
*
|
|
* -- start version control in the template files and add the version number in to the index name, set in options
|
|
* -- push re-indexing into the background with the option background:true (sub-optimal)
|
|
*
|
|
*
|
|
* @author mike@givingassistant.org
|
|
* @version 1.0
|
|
*
|
|
*
|
|
* HISTORY:
|
|
* ========
|
|
* 08-03-17 mks CORE-467: original coding
|
|
* 02-07-19 mks DB-115: fixed index processing
|
|
* 05-21-19 mks DB-116: updated index processing to be specific to each named service environment
|
|
* 10-22-19 mks DB-136: removed assumption that mongo will always be under RBAC allowing unsecured logins
|
|
* Fixed error in $outfile processing where the parent directory is missing
|
|
*
|
|
*/
|
|
$_REDIRECT=false; // enable stdout
|
|
require_once(dirname(__DIR__) . '/config/sneakerstrap.inc'); // load env
|
|
$errors = null;
|
|
$res = 'MDBC: ';
|
|
$envList = null;
|
|
|
|
// get the database configuration
|
|
$dbConfig = gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_MONGODB];
|
|
|
|
// break out of the script if mongo isn't enabled
|
|
if (isset($dbConfig[CONFIG_DATABASE_MONGODB_ENABLED]) and $dbConfig[CONFIG_DATABASE_MONGODB_ENABLED] != 1) {
|
|
exit(ERROR_MDB_NOT_ENABLED);
|
|
}
|
|
|
|
// generate a list of environments this currently-running script can "see"
|
|
foreach (gasConfig::$settings[CONFIG_REGISTERED_SERVICES] as $service => $enabled)
|
|
if (gasConfig::$settings[$service][CONFIG_IS_LOCAL] and gasConfig::$settings[$service][CONFIG_ACTIVE])
|
|
$envList[] = $service;
|
|
|
|
$logFile = dirname(__DIR__) . DIR_LOGS . FILE_MONGO_CONFIG_LOG;
|
|
$outFile = dirname(__DIR__) . DIR_SCRIPTS . DIR_MONGO . FILE_MONGO_INDEX_SCRIPT;
|
|
if (!file_exists($logFile)) touch($logFile);
|
|
if (!file_exists($outFile)) {
|
|
if (!touch($outFile)) {
|
|
$hdr = basename(__FILE__) . AT . __LINE__ . COLON;
|
|
$msg = ERROR_OPEN_LOG_FILE . $outFile;
|
|
consoleLog($res, CON_ERROR, $hdr . $msg);
|
|
consoleLog($res, CON_ERROR, INFO_NO_DIR);
|
|
exit;
|
|
}
|
|
}
|
|
$pLogFile = @fopen($logFile, 'a');
|
|
$pIndexFile = @fopen($outFile, 'w');
|
|
|
|
if (false === $pLogFile) {
|
|
echo ERROR_OPEN_LOG_FILE . $logFile . PHP_EOL;
|
|
exit(1);
|
|
}
|
|
if (false === $pIndexFile) {
|
|
echo ERROR_OPEN_LOG_FILE . $outFile . PHP_EOL;
|
|
exit(1);
|
|
}
|
|
|
|
@fwrite($pLogFile, getDateTime() . CON_SUCCESS . $res . 'begin processing templates' . PHP_EOL);
|
|
@fwrite($pLogFile, getDateTime() . CON_SUCCESS . $res . 'opened output file: ' . $outFile . PHP_EOL);
|
|
@fwrite($pIndexFile, '#! /bin/sh' . PHP_EOL);
|
|
|
|
// pull a copy of the config into memory
|
|
$config = gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_MONGODB];
|
|
$firstPass = true;
|
|
|
|
// loop through each Namaste-supported environment...
|
|
foreach ($envList as $environment) {
|
|
@fwrite($pIndexFile, '# Processing Environment: ' . $environment . PHP_EOL);
|
|
// reset RBAC credentials
|
|
$user = '';
|
|
$password = '';
|
|
$authDB = '';
|
|
$indexText = null;
|
|
// if the current env is set and if that env has enabled mongoDB then...
|
|
if (array_key_exists($environment, $config) and intval($config[$environment][STRING_ENABLED]) === 1) {
|
|
// derive port assignment based-off the current db configuration (shard v. replSet v. standAlone)
|
|
if (intval($config[$environment][CONFIG_DATABASE_MONGODB_SHARDING][CONFIG_DATABASE_MONGODB_ENABLED]) === 1) {
|
|
$port = $config[$environment][CONFIG_DATABASE_MONGODB_SHARDING][CONFIG_DATABASE_MONGODB_SHARDING_MONGOS_NODES][CONFIG_DATABASE_MONGODB_ADMIN_REPLSET_SET];
|
|
$port = intval(substr($port, -5));
|
|
} elseif (intval($config[$environment][CONFIG_DATABASE_MONGODB_REPLSET][CONFIG_DATABASE_MONGODB_REPLSET_ENABLED]) === 1) {
|
|
$port = $config[$environment][CONFIG_DATABASE_MONGODB_REPLSET][CONFIG_DATABASE_MONGODB_REPLSET_DSN][CONFIG_DATABASE_MONGODB_ADMIN_REPLSET_SET][0];
|
|
$port = intval(substr($port, -5));
|
|
} else {
|
|
$port = intval($config[$environment][CONFIG_DATABASE_MONGODB_PORT]);
|
|
}
|
|
// if RBAC is enabled, grabbed the login credentials from the current env
|
|
if ($config[$environment][CONFIG_DATABASE_MONGODB_USE_AUTH] == 1) {
|
|
$user = $config[$environment][CONFIG_DATABASE_MONGODB_USER];
|
|
$password = $config[$environment][CONFIG_DATABASE_MONGODB_PASSWORD];
|
|
$authDB = gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV] . UDASH . $config[$environment][CONFIG_DATABASE_MONGODB_AUTH_SOURCE];
|
|
$command = 'mongo -u ' . $user . ' -p ' . $password . ' --port ' . $port . ' --authenticationDatabase ' . $authDB;
|
|
} else {
|
|
$command = 'mongo --port ' . $port;
|
|
}
|
|
|
|
/** @noinspection PhpComposerExtensionStubsInspection */
|
|
/** @var MongoClient $mongoResource */
|
|
$mongoResource = gasResourceManager::fetchResource(RESOURCE_MONGO_MASTER, $environment);
|
|
|
|
if (is_null($mongoResource)) {
|
|
@fwrite($pLogFile, getDateTime() . CON_ERROR . $res . ERROR_MONGO_CONNECT . AT . $environment . PHP_EOL);
|
|
fclose($pLogFile);
|
|
fclose($pIndexFile);
|
|
exit(1);
|
|
}
|
|
|
|
// STEP 1 -- for every c-type class template, load and process the template
|
|
foreach(glob(dirname(getcwd()) . DIR_CLASSES . DIR_TEMPLATE . '/*' . STRING_CLASS_FILE_EXT) as $filename) {
|
|
$currentClass = basename($filename);
|
|
$currentClass = preg_replace("/" . STRING_CLASS_FILE_EXT . "/", "", $currentClass);
|
|
// extract the $tlti from the front of the template for the factory-class instantiation
|
|
$tlti = substr($currentClass, 0, 3);
|
|
$currentClass = preg_replace("/" . $tlti . "/", "", $currentClass);
|
|
$database = null;
|
|
$collection = null;
|
|
$collectionName = null;
|
|
|
|
if ($currentClass == 'SystemData' and $environment == ENV_ADMIN)
|
|
$x = 1;
|
|
|
|
echo 'processing: ' . $currentClass . '...' . $eos;
|
|
try {
|
|
$e = array();
|
|
$meta = [ META_TEMPLATE => $currentClass, META_CLIENT => CLIENT_SYSTEM, META_TLTI => $tlti ];
|
|
$objFactory = new gacFactory($meta, FACTORY_EVENT_NEW_CLASS, '', $e);
|
|
if (is_null($objFactory)) break; // we couldn't instantiate the factory class for this template
|
|
// do not process PDO scripts
|
|
if ($objFactory->template->schema == TEMPLATE_DB_PDO) {
|
|
$objFactory->__destruct();
|
|
unset($objFactory);
|
|
continue;
|
|
}
|
|
// do not process if the template is not a member of the current environment
|
|
if ($objFactory->template->service != $environment) {
|
|
$objFactory->__destruct();
|
|
unset($objFactory);
|
|
continue;
|
|
}
|
|
$version = $objFactory->template->version;
|
|
$ext = $objFactory->template->extension;
|
|
$service = $objFactory->template->service;
|
|
$collectionName = $objFactory->template->collection;
|
|
/** @var gacMongoDB $tmpObj */
|
|
$tmpObj = $objFactory->widget;
|
|
$collectionName = $tmpObj->getCollectionName();
|
|
$dbName = $tmpObj->getDBName();
|
|
if (is_object($objFactory)) $objFactory->__destruct();
|
|
unset($objFactory);
|
|
} catch (Throwable $t) {
|
|
fclose($pLogFile);
|
|
fclose($pIndexFile);
|
|
exit($t->getMessage() . $eos);
|
|
}
|
|
|
|
$indexes = array(); // this is the index payload we'll submit to the mongoDB Manager\Command
|
|
$options = array(); // this is the options payload for the corresponding index
|
|
$commands = array(); // this is the list of commands that comprise the final ouput
|
|
|
|
// indexes
|
|
$singleIndices = $tmpObj->singleIndexes;
|
|
$compoundIndices = $tmpObj->compoundIndexes;
|
|
$multiKeyIndices = $tmpObj->multiKey;
|
|
// index properties
|
|
$uniqueIndices = $tmpObj->uniqueIndexes;
|
|
$partialIndices = $tmpObj->partialIndexes;
|
|
$t2LiveIndices = $tmpObj->ttlIndexes;
|
|
|
|
// all indexing done in the background
|
|
$sysOption = [
|
|
MONGO_STRING_BACKGROUND => true
|
|
];
|
|
|
|
// order for these two variables is critical: do *NOT* add or delete from one unless you do it to the other
|
|
// while maintaining identical positions for each index type in the arrays. If we ever support indexes
|
|
// other than what appears in these lists, we'll have not only manually add them to the lists, but also
|
|
// code the respective validation and processing bits too.
|
|
$indexTypes = [
|
|
$partialIndices, $uniqueIndices, $t2LiveIndices,
|
|
$singleIndices, $compoundIndices, $multiKeyIndices
|
|
];
|
|
$indexList = [
|
|
'partialIndices' => 0, 'uniqueIndices' => 1, 't2LiveIndices' => 2,
|
|
'singleIndices' => 3, 'compoundIndices' => 4, 'multiKeyIndices' => 5
|
|
];
|
|
|
|
// process partial index properties for the class first
|
|
$count = 0;
|
|
foreach ($indexList as $indexName => $indexCounter) {
|
|
if (!empty($indexTypes[$indexCounter])) {
|
|
$badIndexName = false;
|
|
$indexText[][MONGO_COLLECTION_NAME] = $collectionName;
|
|
// the first part of this is to validate the index column/label names as valid/declared indexes
|
|
foreach ($indexTypes[$indexCounter] as $indexLabel => $indexContents) {
|
|
// strip, if it exists, the extension of the column name so we can grep it in our index lists
|
|
$modifiedField = ($indexName == STRING_PARTIAL_INDICES) ?
|
|
preg_replace("/" . $ext . "/", "", key($indexContents[0]))
|
|
:
|
|
preg_replace("/" . $ext . "/", "", $indexLabel);
|
|
if (!in_array($modifiedField, $tmpObj->indexList)) {
|
|
$hdr = sprintf(INFO_LOC, basename(__FILE__), __LINE__);
|
|
$msg = sprintf(ERROR_MDB_INDEX_UNDECL, array_search($indexCounter, $indexList), basename($filename), $modifiedField);
|
|
echo $hdr . $msg . PHP_EOL;
|
|
$badIndexName = true;
|
|
}
|
|
// if we have one, just one, bad index reference, we need to error-out and stop processing
|
|
// (but we'll echo a list of the bad index names before exiting)
|
|
if ($badIndexName) {
|
|
$hdr = sprintf(INFO_LOC, basename(__FILE__), __LINE__);
|
|
echo $hdr . ERROR_DATA_VALIDATION;
|
|
exit(1);
|
|
}
|
|
/*
|
|
* the $indexes and $options structure contain all elements of the current index. If an index,
|
|
* or index property, does not have options, then the placeholder for that position in $options
|
|
* will be set to null.
|
|
*
|
|
*/
|
|
switch ($indexCounter) {
|
|
// changed structure of $indexes/$options to down-layer the data that will be passed to
|
|
// mongo so as to avoid index name collisions within the arrays using the column name.
|
|
//
|
|
// the goal of this section is to generate the structures that we'll convert to json-
|
|
// -encoded arrays that will ultimately be written to the bash deployment script
|
|
case 0 :
|
|
// partial indexes (done)
|
|
for ($index = 0, $max = count($indexContents[0]); $index < $max; $index++)
|
|
$indexes[][key($indexContents[$index])] = $indexContents[$index][key($indexContents[$index])];
|
|
$options[][ MONGO_STRING_PARTIAL_FE ] = $indexContents[1][0];
|
|
break;
|
|
case 1 :
|
|
// unique index (done)
|
|
$indexes[][$indexLabel] = $uniqueIndices[$indexLabel];
|
|
$options[][MONGO_STRING_UNIQUE] = true;
|
|
break;
|
|
case 2 :
|
|
// time-to-live indexes
|
|
$indexes[][$indexLabel] = 1;
|
|
$options[] = [ MONGO_STRING_EXPIRE_SEC => $t2LiveIndices[$indexLabel] ];
|
|
break;
|
|
case 3 :
|
|
// single indexes (done)
|
|
$indexes[][$indexLabel] = $singleIndices[$indexLabel];
|
|
break;
|
|
case 4 :
|
|
// compound indexes (done)
|
|
// mongo imposes a limit of 32 fields on compound indexes
|
|
// mongo does not use index labels (that's a mysql thing) but we do use the label to
|
|
// distinguish/differentiate the index lists
|
|
$indexes[][$indexLabel] = $indexContents;
|
|
break;
|
|
case 5 :
|
|
// multikey indexes
|
|
for ($index = 0, $max = count($multiKeyIndices); $index < $max; $index++)
|
|
$indexes[][$indexLabel] = $multiKeyIndices[$indexLabel];
|
|
break;
|
|
default :
|
|
// unknown index type
|
|
$hdr = sprintf(INFO_LOC, __FILE__, __LINE__);
|
|
$msg = $hdr . ERROR_MDB_UNK_IDX_TYPE . $indexLabel;
|
|
echo $msg;
|
|
exit (1);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// the second part of this is to generate the BSON for each index type outputting the directives
|
|
// to a bash-script that will be executed (manually) after this script completes processing
|
|
switch ($indexCounter) {
|
|
case 0 :
|
|
// partial indexes
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode($indexes[$index]) . COMMA . json_encode($options[$index]);
|
|
break;
|
|
case 1 :
|
|
// unique indexes (done)
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode($indexes[$index]) . COMMA . json_encode($options[$index]);
|
|
break;
|
|
case 2 :
|
|
// t2Live indexes
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode([$indexes[$index], $options[$index]]);
|
|
break;
|
|
case 3 :
|
|
// single indexes (done)
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode($indexes[$index]);
|
|
break;
|
|
case 4 :
|
|
// compound indexes (done)
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode($indexes[$index][key($indexes[$index])]);
|
|
break;
|
|
case 5 :
|
|
// multi-key indexes
|
|
for ($index = 0, $max = count($indexes); $index < $max; $index++)
|
|
$indexText[] = json_encode($indexes[$index][key($indexes[$index])]);
|
|
break;
|
|
default :
|
|
// unknown index type
|
|
$hdr = sprintf(INFO_LOC, __FILE__, __LINE__);
|
|
$msg = $hdr . ERROR_MDB_UNK_IDX_TYPE . $indexLabel;
|
|
echo $msg;
|
|
exit (1);
|
|
break;
|
|
}
|
|
unset($indexes, $options); // reset the processed-index/options containers
|
|
}
|
|
}
|
|
} // close the env-enabled loop
|
|
if (!empty($indexText)) {
|
|
// we have generated indexing commands --- write out the command to log into mongo
|
|
@fwrite($pIndexFile, $command . ' << EOF' . PHP_EOL);
|
|
@fwrite($pIndexFile, 'use ' . $dbName . PHP_EOL);
|
|
for ($index = 0, $max = count($indexText); $index < $max; $index++) {
|
|
if (is_array($indexText[$index])) {
|
|
$collectionName = $indexText[$index][MONGO_COLLECTION_NAME];
|
|
continue;
|
|
}
|
|
@fwrite($pIndexFile, 'db.' . $collectionName . '.createIndex(' . $indexText[$index] . ')' . PHP_EOL);
|
|
}
|
|
@fwrite($pIndexFile, 'EOF' . PHP_EOL);
|
|
}
|
|
}
|
|
}
|
|
@fwrite($pLogFile, getDateTime() . CON_SUCCESS . $res . basename(__FILE__) . ': script processing ends' . PHP_EOL);
|
|
fclose($pLogFile);
|
|
fclose($pIndexFile);
|
|
chmod($outFile, 0755);
|
|
echo 'processing ended successfully!' . PHP_EOL;
|
|
exit(0);
|