gramps/namaste
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Archive: Namaste PHP AMQP framework v1.0 (2017-2020)

Browse Source 
952 days continuous production uptime, 40k+ tp/s single node.
Original corpo Bitbucket history not included — clean archive commit.
This commit is contained in:
Llama Chile Shop
2026-04-05 09:49:30 -07:00
commit 373ebc8c93
1284 changed files with 409372 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
utilities/ddbtp.php Normal file
View File

@@ -0,0 +1,258 @@
<?php
/**
* DynamoDB Template Processor
*
* in this first version, we're only going to create new tables from the template files...
*
* -- added code to properly handle secondary indices with error checking and verbose error messaging
* -- cannot create a table if it already exists in the database; you must first delete a table
* before creating if if pre-exists unless you're executing this script in a development environment AND
* you've specified the "--delete" command line option.
*
* Delete a table from the command line:
* aws dynamodb delete-table --table-name development_gaLogs --endpoint-url http://localhost:8000
*
* List tables from the command line:
* aws dynamodb list-tables --endpoint-url http://localhost:8000
*
* Describe a table:
* aws dynamodb describe-table --table-name development_gaLogs_log --endpoint-url http://localhost:8000
*
* Help with ddb indexes:
* https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html
*
*
* @author mike@givingassitant.org
* @version 1.0
*
*
* HISTORY:
* ========
* 06-13-17 mks original coding
* 06-27-17 mks processing for secondary indices
* development-env bypass for deleting an existing table
* 06-28-17 mks ensured that attribute keys are not duplicated,
* added check for template type == TEMPLATE_DB_DDB,
* added check that if secondary index, then the hash key is same as base index hash key
* 07-10-18 mks CORE-773: replaced file-logging output with consoleLog()
*
*/
use Aws\DynamoDb\Exception\DynamoDbException;
$_REDIRECT=false; // enable stdout
$opts = getopt("", [ "delete::" ]);
$eos = (isset($_SERVER['HTTP_USER_AGENT'])) ? '<br />' : PHP_EOL;
$topDir = dirname( __DIR__ );
// load the files stored in the common directory
foreach(glob($topDir . '/common/*.php') as $filename) {
/** @noinspection PhpIncludeInspection */
require_once($filename);
}
$classesDir = $topDir . DIR_CLASSES;
$configDir = $topDir . DIR_CONFIG;
$amqpLib = $topDir . DIR_LIB;
$templateDir = $topDir . DIR_CLASSES . DIR_TEMPLATE;
$logDir = $topDir . DIR_LOGS;
date_default_timezone_set(STRING_SYS_TZ);
$eol = "\n";
$res = 'DDBC: ';
/** @noinspection PhpIncludeInspection */
require($topDir . FILE_AUTOLOADER);
if(file_exists($classesDir)) {
Autoloader::register_directory($classesDir);
Autoloader::register_directory($templateDir);
}
require_once $amqpLib . '/vendor/autoload.php';
//load the base config
gasConfig::singleton($configDir . FILE_BASE_CONFIG, FILE_TYPE_XML);
// layer the env config
if (file_exists($configDir . FILE_ENV_CONFIG)) {
gasConfig::addConfig($configDir . FILE_ENV_CONFIG, FILE_TYPE_XML);
}
// environment matters
$validEnvs = [ ENV_DEVELOPMENT, ENV_STAGING, ENV_PRODUCTION ];
$env = gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV];
if (!in_array($env, $validEnvs)) {
consoleLog($res, CON_ERROR, ERROR_ENV_INVALID . $env);
exit(1);
}
// check to see if user is requesting to hard-delete existing table(s)
$hardDelete = (array_key_exists('delete', $opts)) ? true : false; // won't work with a constant
// safety net to ensure that we cannot hard-delete a table in a non-development environment
if ($env != ENV_DEVELOPMENT) $hardDelete = false;
$errors = null;
consoleLog($res, CON_SUCCESS, sprintf(INFO_TEMPLATE_PROCESSING_STARTED, CONFIG_DATABASE_DDB));
/** @var Aws\DynamoDb\DynamoDbClient $ddbConnection */
$ddbConnection = gasResourceManager::fetchResource(RESOURCE_DDB);
if (is_null($ddbConnection)) {
consoleLog($res, CON_ERROR, ERROR_DDB_CONNECT);
exit(1);
}
// first, let's get a list of all the existing tables...
$result = $ddbConnection->listTables();
$meta = $result->get(DDB_METADATA);
if ($meta[DDB_STATUS_CODE] != NUMBER_HTTP_SUCCESS) {
consoleLog($res, CON_ERROR, ERROR_DDB_QUERY . 'listTables()');
exit(1);
}
$tableList = $result->get(DDB_TABLE_NAMES);
if (!sizeof($tableList)) $tableList = null;
$pt = [
DDB_STRING_READ_CAPACITY_UNITS => gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_DDB][CONFIG_DATABASE_READ_CAPACITY_UNITS],
DDB_STRING_WRITE_CAPACITY_UNITS => gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_DDB][CONFIG_DATABASE_WRITE_CAPACITY_UNITS]
];
$counter = 0;
$deleteCounter = 0;
foreach(glob(dirname(__DIR__) . DIR_CLASSES . DIR_TEMPLATE . STRING_CLASS_FILE_GAT . '*' . STRING_CLASS_FILE_EXT) as $filename) {
$currentClass = basename($filename);
$currentClass = preg_replace("/" . STRING_CLASS_FILE_EXT . "/", "", $currentClass);
consoleLog($res, CON_SUCCESS, INFO_PROCESSING . $currentClass);
try {
/** @var gatLogs() $tmpObj */
$tmpObj = new $currentClass();
} catch (Exception $e) {
consoleLog($res, CON_ERROR, $e->getMessage());
}
/** @noinspection PhpUndefinedVariableInspection */
if ($tmpObj->schema != TEMPLATE_DB_DDB) break; // skip if not a DDB template
$iKeys = array_keys($tmpObj->indexes);
$schema = null;
$gsi = null; // global secondary indexes
$lsi = null; // local secondary indexes
$attrDefs = null;
$baseHashName = '';
$uniqAttrList = null;
// process template directives for base index
for ($index = 0; $index < sizeof($tmpObj->indexes); $index++) {
$uniqAttrList[] = $iKeys[$index] . $tmpObj->extension;
if ($tmpObj->indexes[$iKeys[$index]] == DDB_INDEX_HASH) $baseHashName = $iKeys[$index];
$schema[] = [DDB_STRING_ATTRIBUTE_NAME => $iKeys[$index] . $tmpObj->extension, DDB_STRING_KEY_TYPE => $tmpObj->indexes[$iKeys[$index]]];
$attrDefs[] = [DDB_STRING_ATTRIBUTE_NAME => $iKeys[$index] . $tmpObj->extension, DDB_STRING_ATTRIBUTE_TYPE => $tmpObj->fields[$iKeys[$index]]];
}
// process directives for secondary indexes
$indexList = [ DDB_STRING_GSI, DDB_STRING_LSI ];
// loop to process the template's global and then the secondary index array structures
foreach ($indexList as $thisIndex) {
$ti = null; // this index
$requiredFieldList = [STRING_NAME, STRING_INDEXES, DDB_STRING_PT];
$ptValues = [ DDB_PT_KEYS_ONLY, DDB_PT_INCLUDE, DDB_PT_ALL ];
for ($index = 0, $c = count($tmpObj->$thisIndex); $index < $c; $index++) {
$nka = null;
$projection = null;
$keySchema = null;
if (!empty($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE])
and is_array($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE])
) {
$nka = $tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE];
}
if (!in_array($tmpObj->$thisIndex[$index][DDB_STRING_PT], $ptValues)) {
consoleLog($res, CON_ERROR, '(' . $thisIndex . ')::Invalid index projection type: ' . $tmpObj->$thisIndex[$index][DDB_STRING_PT]);
exit(1);
}
$projection = [DDB_STRING_PROJECTION_TYPE => $tmpObj->$thisIndex[$index][DDB_STRING_PT]];
if (!is_null($nka) and $tmpObj->$thisIndex[$index][DDB_STRING_PT] == DDB_PT_INCLUDE) {
$attrList = null;
// append the class extension to the non-key attributes in a projection
foreach ($tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE] as $attribute)
$attrList[] = $attribute . $tmpObj->extension;
$projection[DDB_STRING_NON_KEY_ATTRIBUTES] = $attrList;
// $projection[DDB_STRING_NON_KEY_ATTRIBUTES] = $tmpObj->$thisIndex[$index][DDB_STRING_NON_KEY_ATTRIBUTE];
} elseif (!is_null($nka)) {
consoleLog($res, CON_ERROR, 'Ignored non-key-attribute list because projection type is: ' . $tmpObj->$thisIndex[$index][DDB_STRING_PT]);
}
$ti[$index] = [
DDB_STRING_INDEX_NAME => $tmpObj->$thisIndex[$index][STRING_NAME],
DDB_STRING_PROJECTION => $projection,
];
$c2 = 0;
foreach ($tmpObj->$thisIndex[$index][STRING_INDEXES] as $ik => $iv) {
if ($thisIndex == DDB_STRING_LSI and $iv == DDB_INDEX_HASH and $ik != $baseHashName) {
consoleLog($res, CON_ERROR, 'Template error: a secondary index must have the same HASH key as the base index!');
consoleLog($res, CON_ERROR, 'Check template: ' . $filename . ': ' . $thisIndex . ' and correct');
exit(1);
}
$keySchema[$c2++] = [ DDB_STRING_ATTRIBUTE_NAME => ($ik . $tmpObj->extension), DDB_STRING_KEY_TYPE => $iv ];
if (!in_array(($ik . $tmpObj->extension), $uniqAttrList)) {
$uniqAttrList[] = ($ik . $tmpObj->extension);
$attrDefs[] = [DDB_STRING_ATTRIBUTE_NAME => $ik . $tmpObj->extension, DDB_STRING_ATTRIBUTE_TYPE => $tmpObj->fields[$ik]];
}
}
$ti[$index][DDB_STRING_KEY_SCHEMA] = $keySchema;
if (isset($tmpObj->$thisIndex[$index][STRING_THROUGHPUT]) and $thisIndex == DDB_STRING_GSI) {
$ti[$index][DDB_STRING_PROVISIONED_THROUGHPUT][DDB_STRING_READ_CAPACITY_UNITS] = $tmpObj->$thisIndex[$index][STRING_THROUGHPUT][CONFIG_DATABASE_READ_CAPACITY_UNITS];
$ti[$index][DDB_STRING_PROVISIONED_THROUGHPUT][DDB_STRING_WRITE_CAPACITY_UNITS] = $tmpObj->$thisIndex[$index][STRING_THROUGHPUT][CONFIG_DATABASE_WRITE_CAPACITY_UNITS];
} elseif (isset($tmpObj->$thisIndex[$index][STRING_THROUGHPUT]) and $thisIndex == DDB_STRING_LSI) {
consoleLog($res, CON_ERROR, 'Detected throughput provisioning for local secondary index which is not allowed.');
consoleLog($res, CON_ERROR, 'Check configuration for index: ' . $tmpObj->$thisIndex[$index][STRING_NAME]);
exit(1);
}
}
if ($thisIndex == DDB_STRING_GSI) $gsi = $ti;
if ($thisIndex == DDB_STRING_LSI) $lsi = $ti;
}
$tn = $env . UDASH . $tmpObj->collection . $tmpObj->extension;
$newt = [
DDB_STRING_TABLE_NAME => $tn,
DDB_STRING_KEY_SCHEMA => $schema,
DDB_STRING_ATTRIBUTE_DEFINITIONS => $attrDefs,
DDB_STRING_PROVISIONED_THROUGHPUT => $pt
];
if (!is_null($gsi)) $newt[DDB_STRING_GLOBAL_SI] = $gsi;
if (!is_null($lsi)) $newt[DDB_STRING_LOCAL_SI] = $lsi;
// to be able to delete a table, we must be in development env and explicitly passed the --delete option
$exec = true;
if (in_array($tn, $tableList) and gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV] != ENV_DEVELOPMENT) {
consoleLog($res, CON_ERROR, 'Table: ' . $tn . ' already exists in the database...skipping...');
consoleLog($res, CON_ERROR, 'You must remove a table before you can create it.');
$exec = false;
} elseif (gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV] == ENV_DEVELOPMENT and $hardDelete and in_array($tn, $tableList)) {
consoleLog($res, CON_SYSTEM, 'Table: ' . $tn . ' already exists, but you have authorized it to be deleted.');
try {
$res = $ddbConnection->deleteTable([DDB_STRING_TABLE_NAME => $tn]);
consoleLog($res, CON_SUCCESS, 'Successfully deleted table: ' . $tn);
$deleteCounter++;
} catch (DynamoDbException $e) {
consoleLog($res, CON_ERROR, "Failed to create table: " . $tn);
consoleLog($res, CON_ERROR, $e->getMessage());
exit(1);
} catch (InvalidArgumentException $e) {
consoleLog($res, CON_ERROR, "Failed to create table: " . $tn);
consoleLog($res, CON_ERROR, $e->getMessage());
// print_r($newt);
exit(1);
}
}
if ($exec) {
try {
$res = $ddbConnection->createTable($newt);
consoleLog($res, CON_SUCCESS, 'Successfully created table: ' . $tn);
$counter++;
} catch (DynamoDbException $e) {
consoleLog($res, CON_ERROR, 'Failed to create table: ' . $tn);
consoleLog($res, CON_ERROR, $e->getMessage());
// print_r($newt);
exit(1);
} catch (InvalidArgumentException $e) {
consoleLog($res, CON_ERROR, 'Failed to create table: ' . $tn);
consoleLog($res, CON_ERROR, $e->getMessage());
// print_r($newt);
exit(1);
}
}
}
if ($deleteCounter) {
consoleLog($res, CON_SYSTEM, 'deleted ' . $deleteCounter . ' tables...');
}
consoleLog($res, CON_SUCCESS, 'created ' . $counter . ' tables... ');
exit(0);