gramps
373ebc8c93
952 days continuous production uptime, 40k+ tp/s single node. Original corpo Bitbucket history not included — clean archive commit.
259 lines
12 KiB
PHP
259 lines
12 KiB
PHP
<?php
|
|
/**
|
|
* DynamoDB Template Processor
|
|
*
|
|
* in this first version, we're only going to create new tables from the template files...
|
|
*
|
|
* -- added code to properly handle secondary indices with error checking and verbose error messaging
|
|
* -- cannot create a table if it already exists in the database; you must first delete a table
|
|
* before creating if if pre-exists unless you're executing this script in a development environment AND
|
|
* you've specified the "--delete" command line option.
|
|
*
|
|
* Delete a table from the command line:
|
|
* aws dynamodb delete-table --table-name development_gaLogs --endpoint-url http://localhost:8000
|
|
*
|
|
* List tables from the command line:
|
|
* aws dynamodb list-tables --endpoint-url http://localhost:8000
|
|
*
|
|
* Describe a table:
|
|
* aws dynamodb describe-table --table-name development_gaLogs_log --endpoint-url http://localhost:8000
|
|
*
|
|
* Help with ddb indexes:
|
|
* https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html
|
|
*
|
|
*
|
|
* @author mike@givingassitant.org
|
|
* @version 1.0
|
|
*
|
|
*
|
|
* HISTORY:
|
|
* ========
|
|
* 06-13-17 mks original coding
|
|
* 06-27-17 mks processing for secondary indices
|
|
* development-env bypass for deleting an existing table
|
|
* 06-28-17 mks ensured that attribute keys are not duplicated,
|
|
* added check for template type == TEMPLATE_DB_DDB,
|
|
* added check that if secondary index, then the hash key is same as base index hash key
|
|
* 07-10-18 mks CORE-773: replaced file-logging output with consoleLog()
|
|
*
|
|
*/
|
|
use Aws\DynamoDb\Exception\DynamoDbException;
|
|
|
|
$_REDIRECT=false; // enable stdout
|
|
$opts = getopt("", [ "delete::" ]);
|
|
$eos = (isset($_SERVER['HTTP_USER_AGENT'])) ? '<br />' : PHP_EOL;
|
|
$topDir = dirname( __DIR__ );
|
|
|
|
// load the files stored in the common directory
|
|
foreach(glob($topDir . '/common/*.php') as $filename) {
|
|
/** @noinspection PhpIncludeInspection */
|
|
require_once($filename);
|
|
}
|
|
$classesDir = $topDir . DIR_CLASSES;
|
|
$configDir = $topDir . DIR_CONFIG;
|
|
$amqpLib = $topDir . DIR_LIB;
|
|
$templateDir = $topDir . DIR_CLASSES . DIR_TEMPLATE;
|
|
$logDir = $topDir . DIR_LOGS;
|
|
date_default_timezone_set(STRING_SYS_TZ);
|
|
$eol = "\n";
|
|
$res = 'DDBC: ';
|
|
|
|
/** @noinspection PhpIncludeInspection */
|
|
require($topDir . FILE_AUTOLOADER);
|
|
if(file_exists($classesDir)) {
|
|
Autoloader::register_directory($classesDir);
|
|
Autoloader::register_directory($templateDir);
|
|
}
|
|
require_once $amqpLib . '/vendor/autoload.php';
|
|
|
|
//load the base config
|
|
gasConfig::singleton($configDir . FILE_BASE_CONFIG, FILE_TYPE_XML);
|
|
// layer the env config
|
|
if (file_exists($configDir . FILE_ENV_CONFIG)) {
|
|
gasConfig::addConfig($configDir . FILE_ENV_CONFIG, FILE_TYPE_XML);
|
|
}
|
|
|
|
// environment matters
|
|
$validEnvs = [ ENV_DEVELOPMENT, ENV_STAGING, ENV_PRODUCTION ];
|
|
$env = gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV];
|
|
if (!in_array($env, $validEnvs)) {
|
|
consoleLog($res, CON_ERROR, ERROR_ENV_INVALID . $env);
|
|
exit(1);
|
|
}
|
|
|
|
// check to see if user is requesting to hard-delete existing table(s)
|
|
$hardDelete = (array_key_exists('delete', $opts)) ? true : false; // won't work with a constant
|
|
// safety net to ensure that we cannot hard-delete a table in a non-development environment
|
|
if ($env != ENV_DEVELOPMENT) $hardDelete = false;
|
|
|
|
$errors = null;
|
|
consoleLog($res, CON_SUCCESS, sprintf(INFO_TEMPLATE_PROCESSING_STARTED, CONFIG_DATABASE_DDB));
|
|
|
|
/** @var Aws\DynamoDb\DynamoDbClient $ddbConnection */
|
|
$ddbConnection = gasResourceManager::fetchResource(RESOURCE_DDB);
|
|
|
|
if (is_null($ddbConnection)) {
|
|
consoleLog($res, CON_ERROR, ERROR_DDB_CONNECT);
|
|
exit(1);
|
|
}
|
|
|
|
// first, let's get a list of all the existing tables...
|
|
$result = $ddbConnection->listTables();
|
|
$meta = $result->get(DDB_METADATA);
|
|
if ($meta[DDB_STATUS_CODE] != NUMBER_HTTP_SUCCESS) {
|
|
consoleLog($res, CON_ERROR, ERROR_DDB_QUERY . 'listTables()');
|
|
exit(1);
|
|
}
|
|
$tableList = $result->get(DDB_TABLE_NAMES);
|
|
if (!sizeof($tableList)) $tableList = null;
|
|
|
|
$pt = [
|
|
DDB_STRING_READ_CAPACITY_UNITS => gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_DDB][CONFIG_DATABASE_READ_CAPACITY_UNITS],
|
|
DDB_STRING_WRITE_CAPACITY_UNITS => gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_DDB][CONFIG_DATABASE_WRITE_CAPACITY_UNITS]
|
|
];
|
|
$counter = 0;
|
|
$deleteCounter = 0;
|
|
foreach(glob(dirname(__DIR__) . DIR_CLASSES . DIR_TEMPLATE . STRING_CLASS_FILE_GAT . '*' . STRING_CLASS_FILE_EXT) as $filename) {
|
|
$currentClass = basename($filename);
|
|
$currentClass = preg_replace("/" . STRING_CLASS_FILE_EXT . "/", "", $currentClass);
|
|
consoleLog($res, CON_SUCCESS, INFO_PROCESSING . $currentClass);
|
|
try {
|
|
/** @var gatLogs() $tmpObj */
|
|
$tmpObj = new $currentClass();
|
|
} catch (Exception $e) {
|
|
consoleLog($res, CON_ERROR, $e->getMessage());
|
|
}
|
|
/** @noinspection PhpUndefinedVariableInspection */
|
|
if ($tmpObj->schema != TEMPLATE_DB_DDB) break; // skip if not a DDB template
|
|
$iKeys = array_keys($tmpObj->indexes);
|
|
$schema = null;
|
|
$gsi = null; // global secondary indexes
|
|
$lsi = null; // local secondary indexes
|
|
$attrDefs = null;
|
|
$baseHashName = '';
|
|
$uniqAttrList = null;
|
|
// process template directives for base index
|
|
for ($index = 0; $index < sizeof($tmpObj->indexes); $index++) {
|
|
$uniqAttrList[] = $iKeys[$index] . $tmpObj->extension;
|
|
if ($tmpObj->indexes[$iKeys[$index]] == DDB_INDEX_HASH) $baseHashName = $iKeys[$index];
|
|
$schema[] = [DDB_STRING_ATTRIBUTE_NAME => $iKeys[$index] . $tmpObj->extension, DDB_STRING_KEY_TYPE => $tmpObj->indexes[$iKeys[$index]]];
|
|
$attrDefs[] = [DDB_STRING_ATTRIBUTE_NAME => $iKeys[$index] . $tmpObj->extension, DDB_STRING_ATTRIBUTE_TYPE => $tmpObj->fields[$iKeys[$index]]];
|
|
}
|
|
// process directives for secondary indexes
|
|
$indexList = [ DDB_STRING_GSI, DDB_STRING_LSI ];
|
|
// loop to process the template's global and then the secondary index array structures
|
|
foreach ($indexList as $thisIndex) {
|
|
$ti = null; // this index
|
|
$requiredFieldList = [STRING_NAME, STRING_INDEXES, DDB_STRING_PT];
|
|
$ptValues = [ DDB_PT_KEYS_ONLY, DDB_PT_INCLUDE, DDB_PT_ALL ];
|
|
for ($index = 0, $c = count($tmpObj->$thisIndex); $index < $c; $index++) {
|
|
$nka = null;
|
|
$projection = null;
|
|
$keySchema = null;
|
|
if (!empty($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE])
|
|
and is_array($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE])
|
|
) {
|
|
$nka = $tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE];
|
|
}
|
|
if (!in_array($tmpObj->$thisIndex[$index][DDB_STRING_PT], $ptValues)) {
|
|
consoleLog($res, CON_ERROR, '(' . $thisIndex . ')::Invalid index projection type: ' . $tmpObj->$thisIndex[$index][DDB_STRING_PT]);
|
|
exit(1);
|
|
}
|
|
$projection = [DDB_STRING_PROJECTION_TYPE => $tmpObj->$thisIndex[$index][DDB_STRING_PT]];
|
|
if (!is_null($nka) and $tmpObj->$thisIndex[$index][DDB_STRING_PT] == DDB_PT_INCLUDE) {
|
|
$attrList = null;
|
|
// append the class extension to the non-key attributes in a projection
|
|
foreach ($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE] as $attribute)
|
|
$attrList[] = $attribute . $tmpObj->extension;
|
|
$projection[DDB_STRING_NON_KEY_ATTRIBUTES] = $attrList;
|
|
// $projection[DDB_STRING_NON_KEY_ATTRIBUTES] = $tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE];
|
|
} elseif (!is_null($nka)) {
|
|
consoleLog($res, CON_ERROR, 'Ignored non-key-attribute list because projection type is: ' . $tmpObj->$thisIndex[$index][DDB_STRING_PT]);
|
|
}
|
|
$ti[$index] = [
|
|
DDB_STRING_INDEX_NAME => $tmpObj->$thisIndex[$index][STRING_NAME],
|
|
DDB_STRING_PROJECTION => $projection,
|
|
];
|
|
$c2 = 0;
|
|
foreach ($tmpObj->$thisIndex[$index][STRING_INDEXES] as $ik => $iv) {
|
|
if ($thisIndex == DDB_STRING_LSI and $iv == DDB_INDEX_HASH and $ik != $baseHashName) {
|
|
consoleLog($res, CON_ERROR, 'Template error: a secondary index must have the same HASH key as the base index!');
|
|
consoleLog($res, CON_ERROR, 'Check template: ' . $filename . ': ' . $thisIndex . ' and correct');
|
|
exit(1);
|
|
}
|
|
$keySchema[$c2++] = [ DDB_STRING_ATTRIBUTE_NAME => ($ik . $tmpObj->extension), DDB_STRING_KEY_TYPE => $iv ];
|
|
if (!in_array(($ik . $tmpObj->extension), $uniqAttrList)) {
|
|
$uniqAttrList[] = ($ik . $tmpObj->extension);
|
|
$attrDefs[] = [DDB_STRING_ATTRIBUTE_NAME => $ik . $tmpObj->extension, DDB_STRING_ATTRIBUTE_TYPE => $tmpObj->fields[$ik]];
|
|
}
|
|
}
|
|
$ti[$index][DDB_STRING_KEY_SCHEMA] = $keySchema;
|
|
if (isset($tmpObj->$thisIndex[$index][STRING_THROUGHPUT]) and $thisIndex == DDB_STRING_GSI) {
|
|
$ti[$index][DDB_STRING_PROVISIONED_THROUGHPUT][DDB_STRING_READ_CAPACITY_UNITS] = $tmpObj->$thisIndex[$index][STRING_THROUGHPUT][CONFIG_DATABASE_READ_CAPACITY_UNITS];
|
|
$ti[$index][DDB_STRING_PROVISIONED_THROUGHPUT][DDB_STRING_WRITE_CAPACITY_UNITS] = $tmpObj->$thisIndex[$index][STRING_THROUGHPUT][CONFIG_DATABASE_WRITE_CAPACITY_UNITS];
|
|
} elseif (isset($tmpObj->$thisIndex[$index][STRING_THROUGHPUT]) and $thisIndex == DDB_STRING_LSI) {
|
|
consoleLog($res, CON_ERROR, 'Detected throughput provisioning for local secondary index which is not allowed.');
|
|
consoleLog($res, CON_ERROR, 'Check configuration for index: ' . $tmpObj->$thisIndex[$index][STRING_NAME]);
|
|
exit(1);
|
|
}
|
|
}
|
|
if ($thisIndex == DDB_STRING_GSI) $gsi = $ti;
|
|
if ($thisIndex == DDB_STRING_LSI) $lsi = $ti;
|
|
}
|
|
$tn = $env . UDASH . $tmpObj->collection . $tmpObj->extension;
|
|
$newt = [
|
|
DDB_STRING_TABLE_NAME => $tn,
|
|
DDB_STRING_KEY_SCHEMA => $schema,
|
|
DDB_STRING_ATTRIBUTE_DEFINITIONS => $attrDefs,
|
|
DDB_STRING_PROVISIONED_THROUGHPUT => $pt
|
|
];
|
|
if (!is_null($gsi)) $newt[DDB_STRING_GLOBAL_SI] = $gsi;
|
|
if (!is_null($lsi)) $newt[DDB_STRING_LOCAL_SI] = $lsi;
|
|
|
|
// to be able to delete a table, we must be in development env and explicitly passed the --delete option
|
|
$exec = true;
|
|
if (in_array($tn, $tableList) and gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV] != ENV_DEVELOPMENT) {
|
|
consoleLog($res, CON_ERROR, 'Table: ' . $tn . ' already exists in the database...skipping...');
|
|
consoleLog($res, CON_ERROR, 'You must remove a table before you can create it.');
|
|
$exec = false;
|
|
} elseif (gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV] == ENV_DEVELOPMENT and $hardDelete and in_array($tn, $tableList)) {
|
|
consoleLog($res, CON_SYSTEM, 'Table: ' . $tn . ' already exists, but you have authorized it to be deleted.');
|
|
try {
|
|
$res = $ddbConnection->deleteTable([DDB_STRING_TABLE_NAME => $tn]);
|
|
consoleLog($res, CON_SUCCESS, 'Successfully deleted table: ' . $tn);
|
|
$deleteCounter++;
|
|
} catch (DynamoDbException $e) {
|
|
consoleLog($res, CON_ERROR, "Failed to create table: " . $tn);
|
|
consoleLog($res, CON_ERROR, $e->getMessage());
|
|
exit(1);
|
|
} catch (InvalidArgumentException $e) {
|
|
consoleLog($res, CON_ERROR, "Failed to create table: " . $tn);
|
|
consoleLog($res, CON_ERROR, $e->getMessage());
|
|
// print_r($newt);
|
|
exit(1);
|
|
}
|
|
}
|
|
if ($exec) {
|
|
try {
|
|
$res = $ddbConnection->createTable($newt);
|
|
consoleLog($res, CON_SUCCESS, 'Successfully created table: ' . $tn);
|
|
$counter++;
|
|
} catch (DynamoDbException $e) {
|
|
consoleLog($res, CON_ERROR, 'Failed to create table: ' . $tn);
|
|
consoleLog($res, CON_ERROR, $e->getMessage());
|
|
// print_r($newt);
|
|
exit(1);
|
|
} catch (InvalidArgumentException $e) {
|
|
consoleLog($res, CON_ERROR, 'Failed to create table: ' . $tn);
|
|
consoleLog($res, CON_ERROR, $e->getMessage());
|
|
// print_r($newt);
|
|
exit(1);
|
|
}
|
|
}
|
|
}
|
|
if ($deleteCounter) {
|
|
consoleLog($res, CON_SYSTEM, 'deleted ' . $deleteCounter . ' tables...');
|
|
}
|
|
consoleLog($res, CON_SUCCESS, 'created ' . $counter . ' tables... ');
|
|
exit(0);
|