<?php

use MongoDB\Driver\Cursor;
use MongoDB\Driver\ReadPreference as ReadPreferenceAlias;
use MongoDB\Driver\WriteConcern as WriteConcernAlias;

class gacMongoDB extends gaaNamasteCore
{
    private ?object $manager = null;// mongoDB manager class object (database connection)
    private string $dbName;         // container for the DB name defined in the XML configuration
    private string $nameSpace;      // combination of the DB name and the collection name defined in the template
    private bool $journaling;       // boolean indicating if mongo journaling is enabled (XML config value)
    private ReadPreferenceAlias $readPreference;    // the db read preference pulled from the XML (master/slave thing)
    private int $wtimeout;          // write timeout value from the XML config
    private WriteConcernAlias $writeConcern;        // sets the write concern params for the object constructor
//    private $result;              // stores MongoDB WriteResult class object
    private bool $limitOverride;    // used to over-ride the system-limit on query returns -- only used internally
    private string $env;            // set's the environment name from the XML config for setting table names
    private string $res;            // for console log output
    public string $migrationSortKey;// holds the migration sort key string, if defined
    public array $migrationStatusKV;// holds the associative array for the migration status key->value pair, if defined

    /**
     * gacMongoDB constructor.
     *
     * The gacMongoDB constructor is an instantiation class that is invoked by the gacFactory class based on the
     * requested data class template schema defined for the requested class.
     *
     * This class is responsible for all i/o with mongoDB - validating data according to the declarations in the class
     * template, building queries, and accessing the HVVM MongoDB drivers for CRUD operations.
     *
     * The constructor class handles initialization of the data class:
     *     -- sets the eventGUID as extracted from the meta data payload
     *     -- loads the meta data and sets members based on various meta data members
     *     -- instantiates the parent (Namaste Core) class abstraction
     *     -- fetches a MongoDB resource from the resource manager
     *     -- loads the data template
     *     -- loads MongoDB internals (write concerns, read preferences, collections and namespaces)
     *     -- loads a MongoDB record if the GUID is included in the input parameters
     *     -- instantiates logging under the event GUID
     *
     * to-do work:
     * -----------
     *     -- add the record-load feature functionality
     *     -- add auditing
     *     -- add journaling
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_meta
     * @param string $_id
     *
     * HISTORY:
     * ========
     * 07-14-17     mks     CORE-464:  initial coding
     * 09-08-17     mks     CORE-529:  passing event GUID down to the logger sub-class
     * 10-03-17     mks     CORE-572:  updated for new read-preference values/processing
     * 11-28-17     mks     CORE-635:  fixed: was using tagging for write-concern on replSets when tagging not
     *                                 yet implemented and was throwing exception on destructive operations
     * 02-07-18     mks     _INF-139:  PHP 7.2 exception handling, console logging of exception
     * 03-01-18     mks     CORE-689:  refactored env tagging -- env tags are now prepended to the database name
     *                                 and were removed from the collection name.  Replaced last echo w/consoleLog.
     * 03-02-18     mks     CORE-680:  deprecated trace logging
     * 03-12-18     mks                Fixed a PHP log warning generated while assigning the $data count
     * 06-08-18     mks     CORE-1035: deprecated $this->migration map for new migration config container
     * 08-01-18     mks     CORE-774:  PHP7.2 exception handling
     * 11-13-18     mks     DB-51:     constructor will now load a record during instantiation if passed as a parameter
     * 11-27-18     mks     DB-51:     streamlined the code that fetches a record if guid is provided as a parameter and
     *                                 exception-wrapped the call to _fetchRecords()
     * 04-26-19     mks     DB-116:    improved error reporting @ system level by adding location data to console and
     *                                 internal log messages.
     * 01-08-20     mks     DB-150:    PHP7.4 class member type-casting
     * 04-05-20     mks     ECI-136:   changed w=2 for connections s.t. update-fetch flows work successfully for fetch
     *
     */
    public function __construct(array $_meta, string $_id = '')
    {
        register_shutdown_function([$this, STRING_DESTRUCTOR]);
        $this->res = 'cMDB: ';

        // instantiate the parent abstraction class
        try {
            parent::__construct($_meta[META_EVENT_GUID] ?? null);
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            consoleLog($this->res, CON_ERROR, $hdr . $msg);
            return;
        }

        // define acceptable read-preferences catalog: http://php.net/manual/en/class.mongodb-driver-readpreference.php
        $readPreferences = [
            PDO_RP_PRIMARY => 1,
            PDO_RP_PRIMARY_PREFERRED => 5,
            PDO_RP_SECONDARY => 2,
            PDO_RP_SECONDARY_PREFERRED => 6,
            PDO_RP_NEAREST => 10
        ];

        // store meta data and client identifier first because we'll need this in the parent constructor
        $this->templateClass = $_meta[META_TEMPLATE];
        $this->client = STRING_UNDEFINED;
        if (!empty($_meta)) {
            $this->metaPayload = $_meta;
            if (isset($this->metaPayload[META_CLIENT])) {
                $this->client = $this->metaPayload[META_CLIENT];
            }
        }
        if (isset($this->metaPayload[META_EVENT_GUID])) {
            $this->eventGUID = $this->metaPayload[META_EVENT_GUID];
        } else {
            $this->eventMessages[] = ERROR_EVENT_GUID_404 . $this->class;
        }

        if (!isset($this->logger)) {
            try {
                $this->logger = new gacErrorLogger($_meta[META_EVENT_GUID] ?? null);
            } catch (Throwable $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
                $this->eventMessages[] = $msg;
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
                return;
            }
        }

        $errors = null;
        $this->status = false;
        $this->limitOverride = false;
        $this->env = gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV];

        if (empty(gasResourceManager::$cfgMongo)) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_CONFIG_RESOURCE_404 . RESOURCE_MONGO_MASTER;
            $this->logger->error($hdr . $msg);
            $this->eventMessages[] = $msg;
            $this->setState(STATE_RESOURCE_ERROR_MONGO);
            return;
        }
        $this->config = gasResourceManager::$cfgMongo;

        $template = dirname(__DIR__) . DIR_CLASSES . DIR_TEMPLATE . STRING_CLASS_FILE_GAT . $_meta[META_TEMPLATE] . STRING_CLASS_FILE_EXT;
        if (!file_exists($template)) {
            $this->logger->warn(ERROR_FILE_404 . $template);
            try {
                $this->setState(ERROR_CLASS_404 . $_meta[META_TEMPLATE]);
            } catch (TypeError $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $msg = ERROR_TYPE_EXCEPTION . COLON . $t->getMessage();
                $this->eventMessages[] = $msg;
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            }
            return;
        }

        try {
            // if we don't have a tlti set in the meta payload then the instantiation request originated outside of
            // the SMAX API then we can assume the GA TLTI as a default
            $templateName = (isset($_meta[META_TLTI])) ? $_meta[META_TLTI] . $_meta[META_TEMPLATE] : STRING_CLASS_GAT . $_meta[META_TEMPLATE];
            if (!$this->loadTemplates($templateName)) {
                $this->state = null;
                $this->logger->warn(ERROR_CLASS_404 . $_meta[META_TLTI] . $_meta[META_TEMPLATE]);
                if (empty($this->state)) {
                    $this->setState(STATE_CLASS_INSTANTIATION_ERROR);
                }
                return;
            }
            if (!$this->logger->setService($this->dbService)) {
                $this->eventMessages[] = ERROR_SERVICE_404 . $this->dbService;
                $this->logger->fatal(sprintf(INFO_LOC, basename(__FILE__), __LINE__) . ERROR_SERVICE_404 . $this->dbService);
                return;
            }
        } catch (TypeError $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_TYPE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->warn($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            return;
        }

        try {
            $this->manager = gasResourceManager::fetchResource(RESOURCE_MONGO_MASTER, $this->dbService);
            if (is_null($this->manager)) {
                $this->logger->warn(ERROR_SERVICE_404 . RESOURCE_MONGO_MASTER . AT . $this->dbService);
                $this->setState(STATE_RESOURCE_ERROR_MONGO);
                return;
            }
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->warn($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            return;
        }

        // set-up properties for the mongodb write-concern object
        $this->journaling = (intval($this->config[$this->dbService][CONFIG_DATABASE_MONGODB_JOURNAL]) == 1) ? true : false;
        $this->wtimeout = intval($this->config[$this->dbService][CONFIG_DATABASE_MONGODB_WTO]);  // todo: sanity check

        // instantiate the mongodb write-concern object based on the current environment and configuration
        try {
            // ECI-136: set the write-concern to majority - this should handle most post-write fetches from read-slaves
            //          and accommodate varying replication-set configurations.  This has been tested and proven as a
            //          solution so please don't change unless you're absolutely sure you know why you're doing it.
            $this->writeConcern = new MongoDB\Driver\WriteConcern(MongoDB\Driver\WriteConcern::MAJORITY, $this->wtimeout, $this->journaling);
            
            // set the db name and the namespace
            if (empty($this->env)) $this->env = gasConfig::$settings[CONFIG_ID][CONFIG_ID_ENV];
            $this->dbName = $this->env . UDASH . $this->config[$this->dbService][CONFIG_DATABASE_MONGODB_DB_NAME];
            $this->nameSpace = $this->dbName . DOT . $this->collectionName;     // collection name set in loadTemplate()

            // set the read preference dependent on the env for the current class
            $rp = $this->config[$this->dbService][CONFIG_DATABASE_MONGODB_SECONDARY_RP];
            if (array_key_exists($rp, $readPreferences)) {
                $rpValue = $readPreferences[$rp];
            } else {
                $msg = sprintf(ERROR_MDB_INVALID_RP, $rp);
                $this->logger->fatal($msg);
                $this->eventMessages[] = $msg;
                $this->state = STATE_FRAMEWORK_FAIL;
                return;
            }
            $this->readPreference = new MongoDB\Driver\ReadPreference($rpValue);
        } catch (MongoDB\Driver\Exception\InvalidArgumentException |
                 Throwable $e) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = $e->getMessage();
            $this->eventMessages[] = $msg;
            $this->logger->warn($hdr . $msg);
            consoleLog($this->res, CON_SYSTEM, $hdr . $msg);
            $this->state = STATE_DB_ERROR;
            return;
        }

        if (!empty($_id)) {
            if (!validateGUID($_id)) {
                $msg = ERROR_INVALID_GUID . $_id;
                $this->logger->data($msg);
                $this->state = STATE_GUID_ERROR;
                return;
            }
            $tweakedPerms = false;                  // did we override the audit controls for the fetch?
            $oldAudit = $this->skipReadAudit;       // save the old setting b/c we'll override if audit is enabled
            // DB-51: populating a data class on instantiation if GUID passed to constructor
            $query = [ DB_TOKEN => [ OPERAND_NULL => [ OPERATOR_EQ => [ $_id ]]]];
            // if this is a system request, disable auditing/journaling for the fetch only
            if ($this->useAuditing == AUDIT_NONDESTRUCTIVE and $this->metaPayload[META_CLIENT] == CLIENT_SYSTEM) {
                $this->skipReadAudit = true;
                $tweakedPerms = true;
            }
            try {
                $this->_fetchRecords([STRING_QUERY_DATA => $query]);
            } catch (TypeError $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $this->eventMessages[] = sprintf(STUB_LOC, basename(__FILE__), __METHOD__, __LINE__) . ERROR_TYPE_EXCEPTION;
                $this->eventMessages[] = $t->getMessage();
                $this->logger->error($hdr . $t->getMessage());
                $this->state = STATE_FRAMEWORK_FAIL;
                $this->status = false;
                return;
            }
            if ($tweakedPerms) $this->skipReadAudit = $oldAudit;
        }
        $this->queryCount = 0;
        $this->count = (!empty($this->data)) ? count($this->data) : 0;
//        $this->doRollback = false;
//        $this->rollBackEvents = null;
//        $this->rollBackQueries = null;
        $this->status = true;
        $this->state = STATE_SUCCESS;
    }


    /**
     * createRecord() -- public method
     *
     * This is one of the abstracted-defined CRUD methods that will handle new record creation for mongo schemas.
     *
     * There are two input parameters to this method, again - as defined in it's declarative statement in the core
     * abstraction, that are the payload data and a string to help with processing, respectively.
     *
     * $_data -- an array of associative data tuples [ [ tuple1, ..., tupleN ] ]
     * $_prevalidated -- defines the data scope wrt/processing
     *
     * The method validates the input parameters, logging an error and generating diagnostic messages if validation
     * fail (also returning control to the invoking client).
     *
     * After validation is complete, we invoke a private method to load the two params, passing in the create event
     * as well.  this effectively loads the data array into the current class member and, for every tuple in the data
     * payload, also assigns the create event history record (meta data) as a sub-element array.
     *
     * Next, we check to see if there already exists a sequence (PKey) value in each tuple - if one exists, then
     * we're going to end processing as the calling client mistakenly called create instead of update.
     *
     * Next, we generate the sequence key (similar to mysql's autoincrement) for each $data tuple and assign it.  If
     * the current class supports Tokens, then we'll generate and populate a token for each tuple as well.
     *
     * If there is only one tuple in $data, we're going to call the mongo Save() api command where if there exists
     * more than one tuple in $data, then we're going to call the mongo batchInsert() api command to insert (create)
     * all of the tuples at one time.
     *
     * In both cases, the return codes are assigned to the class $queryResults parameter.  Both mongo api calls
     * are extensively exception-wrapped so any error thrown will be trapped and logged.
     *
     * After the successful api call, if the class supports caching, and there exists a defined cacheMap in the
     * current class, then we're going to pre-cache all the newly created records under the assumption that
     * subsequent calls to ADS will request these just-created records.
     *
     * results, state/status, and diagnostic messages, are all stored in the class member variables assigned to these
     * respective tasks.  The method itself is type void so it's up to the calling client to validate the success
     * of this method via the aforementioned class member variables.
     *
     * NOTES:
     * ======
     * for future development, consider the following features:
     * -- submitted tuples that have already been saved can be processed by the update method instead of causing
     *    the request to be summarily rejected.
     * -- generated tokens can be checked for duplicates in the existing collection and can be regenerated to ensure
     *    that tokens remain unique despite the extremely low probability that a duplicate GUID can be generated.
     * -- tokenList is built in prepareBulkWriteInsertData()
     *
     * todo -- extract sub-collection data from payload and process as sub-collections using the named methods
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     * @param string $_preValidated  must be one of three constant values: DATA_NORM (default), DATA_MIG, or DATA_WH
     *
     *
     * HISTORY:
     * ========
     * 07-14-17     mks     CORE-464: initial coding
     * 08-18-17     mks     CORE-500: fixed bug in conditional for adding token to payload
     * 09-06-17     mks     CORE-556: injecting guids into sub-collection records
     * 09-08-17     mks     CORE-529: added eventGUID to data records
     * 10-20-17     mks     CORE-585: refactored - moved generic stuffs to core making everything more oopy
     * 02-06-18     mks     INF-139:  moved the bulk-write mongo code out to a protected method so that it can
     *                                be accessed w/out all the data validation in this method, added a bypass
     *                                to the loadPayloadData() method for pre-validated (migration) data so that
     *                                we're not redundantly validating.
     * 04-28-18     mks     INF-188:  support for meta_limit_override in warehousing operations
     * 09-01-18     mks     DB-48:    bypassing loadPayloadData() for migration/warehouse payloads
     * 10-17-18     mks     DB-59:    Auditing code added
     * 02-15-19     mks     DB-116:   removed caching code - deprecated; cache mapping happening @ broker service level
     * 09-25-20     mks     DB-168:   containerized legit-sources for bypassing new payload validation, adding user
     *                                data to the list b/c user data is pre-validated in the user class
     * 12-11-20     mks     DB-180:   Support for CONS data
     *
     */
    public function _createRecord(array $_data, string $_preValidated = DATA_NORM): void
    {
        $legitSources = [ DATA_MIGR, DATA_SESS, DATA_USER ];  // that isn't audit or cons data
        try {
            if ($_preValidated == DATA_WHSE) {
                if (!$this->addData($_data)) {
                    $this->state = STATE_DATA_ERROR;
                    $this->status = false;
                    return;
                }
            } elseif ($_preValidated == DATA_AUDT) {
                // audit or CONS event -- if pre-validated data is not already loaded, then copy
                $row = 0;
                foreach ($_data as $record) {
                    foreach ($record as $field => $value) {
                        $this->data[$row][$field . $this->ext] = $value;
                    }
                    $row++;
                }
            } elseif ($_preValidated == DATA_CONS) {
                // if template is not local to the executing env, make this a broker event request to the target service
                if (!gasConfig::$settings[CONFIG_BROKER_SEGUNDO][CONFIG_IS_LOCAL]) {
                    $payload = [
                        BROKER_REQUEST => BROKER_REQUEST_CONS,
                        BROKER_DATA => $_data,
                        BROKER_META_DATA => $this->metaPayload,
                        STRING_SERVICE => ENV_APPSERVER
                    ];
                    if (!validateMetaData($payload, $this->eventMessages)) {
                        $this->state = STATE_DATA_ERROR;
                        $this->status = false;
                        return;
                    }
                } else {
                    $this->data = $_data;
                    $this->count = count($this->data);
                }
            } elseif (in_array($_preValidated, $legitSources)) {
                // data was pre-validated during fetch from the remote source
                $this->data = $_data;
            } else {
                // transfer the client-data payload into the class $data property with validation
                // if the method returns a false, a processing error happened so return immediately
                // note that data-array validation is handled in this method...
                try {
                    if (!$this->loadPayloadData($_data, DB_EVENT_CREATE)) return;
                } catch (TypeError $t) {
                    $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                    $msg = sprintf(INFO_LOC, basename(__FILE__), __LINE__) . ERROR_TYPE_EXCEPTION;
                    $this->logger->warn($hdr . $msg);
                    $this->logger->warn($t->getMessage());
                    $this->eventMessages[] = $msg;
                    $this->eventMessages[] = $t->getMessage();
                    $this->status = false;
                    $this->state = STATE_DATA_TYPE_ERROR;
                    return;
                }
            }
            // reset state/status members
            $this->state = STATE_DATA_ERROR;
            $this->status = false;

            // inject record GUIDs into each payload record
            if (!$this->newRecordDataInjections()) return;

            // forced validation of the the number of tuples in the current container
            if (count($this->data) != $this->count) {
                $this->logger->warn(ERROR_DATA_INCONSISTENT_COUNT);
                $this->count = count($this->data);
            }

            $this->state = STATE_DB_ERROR;
            $this->status = false;
            if ($this->count == 0) {
                $this->logger->info(ERROR_DATA_ARRAY_EMPTY);
                return;
            }

            // allow for meta-limit override for warehousing
            if (!isset($this->metaPayload[META_LIMIT_OVERRIDE])) {
                // ensure we're not inserting more than the max # of records allowed b/c batch processing eats memory
                if ($this->count > $this->recordLimit) {
                    $msg = ERROR_RECORD_LIMIT_EXCEEDED . $this->recordLimit;
                    $this->eventMessages[] = $msg;
                    if ($this->debug) $this->logger->debug($msg);
                    $this->state = STATE_DATA_ERROR;
                    return;
                }
            } elseif (1 !== intval($this->metaPayload[META_LIMIT_OVERRIDE])) {
                $msg = ERROR_DATA_META_REJECTED . META_LIMIT_OVERRIDE;
                $this->eventMessages[] = $msg;
                if ($this->debug) $this->logger->debug($msg);
                $this->state = STATE_META_ERROR;
                return;
            } elseif (1 === intval($this->metaPayload[META_LIMIT_OVERRIDE])) {
                $this->setRecordLimit(1000);  // no limit
            } else {
                $this->setRecordLimit(gasConfig::$settings[CONFIG_BROKER_SERVICES][CONFIG_WH_RECS_PER_XFER]);
            }
            $this->event = DB_EVENT_CREATE;
            // initialize the bulk-write object, insert the data, and save to the class collection
            $this->prepareBulkWriteInsertData();

            // if we successfully created a record(s), process the return data set
            if ($this->status) {
                // DB-59: check to see if auditing is enabled and create the audit payload if necessary
                //        we want to do this before we filter the data currently stored as a member variable
                if ($this->useAuditing) {
                    // if the audit event fails, we'll continue processing without raising an error (to the client)
                    if (!$this->registerAuditEvent(EVENT_NAME_AUDIT_CREATE)) {
                        $this->eventMessages[] = ERROR_AUDIT_GENERIC_FAIL;
                        consoleLog($this->res, CON_ERROR, ERROR_AUDIT_GENERIC_FAIL);
                    }
                }
                if ($_preValidated == DATA_NORM) {
                    if (!$this->returnFilteredData())
                        $this->eventMessages[] = ERROR_DATA_PROCESSING;
                }
                /*
                 * NOTE:  for non-DATA_NORM classes, which is pretty much limited to data-warehousing classes, none
                 *        of which have caching enabled, we're ok NOT building a token list.
                 */
            }
        } catch (MongoDB\Driver\Exception\ConnectionTimeoutException | Throwable | TypeError $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $this->eventMessages[] = ERROR_EXCEPTION;
            @handleExceptionMessaging($hdr, $t->getMessage(), $foo, true);
            $this->state = STATE_FRAMEWORK_WARNING;
        }
    }


    /**
     * _fetchRecords() -- core-declared public method
     *
     * This method is declared in the core -- it is the READ part of the supported CRUD events returning data, by
     * request, back to the requesting client.
     *
     * The input parameter to the method is the BROKER_DATA payload as received by the Namaste event broker.  This
     * data array should carry definition for STRING_QUERY_DATA and may also have arrays for STRING_SORT_DATA and
     * STRING_RETURN_DATA.  Caching, skip and limit directives are embedded in the meta-data array already assigned
     * to a class property.
     *
     * The method works by invoking a series of class method that:
     *
     * -- builds the query based on the BROKER_DATA -> STRING_QUERY_DATA payload
     * -- builds the sort based on the BROKER_DATA -> STRING_SORT_DATA payload, if available
     * -- builds the return data (the projection) based on the BROKER_DATA -> STRING_RETURN_DATA payload, if available.
     * -- builds the cacheMap/exposedFields/CleanData return data set.
     *
     * Results are returned to the calling client via the:
     *
     * state property or status property or $data property -- regardless, it is the responsibility of the calling client
     * to check the various properties to ensure correct processing continuation.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     *
     * HISTORY:
     * ========
     * 07-17-17     mks     CORE-464: initial coding
     * 08-07-17     mks     CORE-497: support for over-riding the query limit
     * 08-14-17     mks     CORE-493: fixed issue with record count (lost the env from the collection name)
     * 08-15-17     mks     CORE-487: injecting status filter on classes with soft-deletes
     * 08-24-17     mks     CORE-494: added processing for no-data found setting state = STATE_NOT_FOUND
     * 09-26-17     mks     CORE-572: hard-delete parse/status injection into query moved to core
     * 10-20-17     mks     CORE-585: changed recordLimit to a member variable (defined in core)
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging
     * 03-12-18     mks     Fixed PHP log warning when assigning count($data) to member var
     * 03-14-18     mks     CORE-833: condensing debug output to a single event call on query building
     * 04-28-18     mks     _INF-188: support for meta_limit_override for warehousing
     * 10-29-18     mks     DB-68: support for AUDIT_NONDESTRUCTIVE setting
     * 12-03-18     mks     DB-55: using cursor->setTypeMap() to force return values as arrays (instead of objects)
     *                             which deprecated the deBSON() function call.
     *
     */
    public function _fetchRecords(array $_data):void
    {
        $this->status = false;
        $this->state = STATE_DATA_ERROR;
        $sort = null;
        $startTime = floatval(0);

        // auditing
        // not going to file a system event for missing meta b/c, w/out meta, we can't track the request back
        if ($this->useAuditing == AUDIT_NONDESTRUCTIVE and (empty($this->metaPayload) or !is_array($this->metaPayload))) {
            $this->state = STATE_META_ERROR;
            $this->logger->data(ERROR_DATA_META_REQUIRED);
//echo '***** META DATA MISSING ERROR *****' . COLON . __LINE__ . PHP_EOL;
//            echo 'This->class: ' . $this->class . PHP_EOL;
//            echo 'this->auditLevel = ' . $this->useAuditing . PHP_EOL;
//echo '***** META DATA MISSING ERROR *****' . COLON . __LINE__ . PHP_EOL;
            return;
        }

        $query = isset($_data[STRING_QUERY_DATA]) ? $_data[STRING_QUERY_DATA] : [];
        $oldQuery = $query;
        list($query, $injection) = $this->softDeleteStatusInjection($query);

        try {
            // build and submit the query
            $query = $this->queryBuilder($query);
            if (!$this->status) {
                if (!$injection) {
                    // we didn't inject a query so log the failure and return
                    $this->eventMessages[] = ERROR_DATA_QUERY_BUILD;
                    $this->logger->error(ERROR_DATA_QUERY_BUILD . COLON . $this->collectionName . COLON . json_encode($query));
                    return;
                } elseif (!empty($oldQuery)) {
                    // attempt to build the original query
                    $query = $this->queryBuilder($oldQuery);
                    if (!$this->status) {
                        $this->eventMessages[] = ERROR_DATA_QUERY_BUILD;
                        $this->logger->error(ERROR_DATA_QUERY_BUILD . COLON . $this->collectionName . COLON . json_encode($oldQuery));
                        return;
                    }
                }
            }

            // reset from queryBuilder()
            $this->status = false;
            $this->state = STATE_DATA_ERROR;

            // check sort string -- if not empty, then build the sort directive
            if (array_key_exists(STRING_SORT_DATA, $_data) and !empty($_data[STRING_SORT_DATA]) and is_array($_data[STRING_SORT_DATA])) {
                $sort = $this->sortBuilder($_data[STRING_SORT_DATA]);
                if (is_null($sort)) return;
            } elseif (array_key_exists(STRING_SORT_DATA, $_data)) {
                $this->eventMessages[] = ERROR_MDB_SORT_404;
                return;
            }

            // build the projection list
            $projection = (array_key_exists(STRING_RETURN_DATA, $_data) and !empty($_data[STRING_RETURN_DATA])) ? $this->projectionBuilder($_data[STRING_RETURN_DATA]) : $this->projectionBuilder(array());
            if (!empty($sort)) $this->strQuery .= '.sort(' . json_encode($sort) . ')';
            if (!empty($sort)) $options[STRING_SORT] = $sort;
            $options[STRING_PROJECTION] = $projection;
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            return;
        }


        // process skip/limit values
        // first, check to see if the limit override has been set
        $metaLimit = intval(gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT]);
        $this->limitOverride = (array_key_exists(META_LIMIT_OVERRIDE, $this->metaPayload) and $this->metaPayload[META_LIMIT_OVERRIDE] === 1) ? true : false;
        $skip = (array_key_exists(META_SKIP, $this->metaPayload)) ? intval($this->metaPayload[META_SKIP]) : 0;
        // if limit was passed in-Meta, set to the meta value, else set to the system max
        $limit = (array_key_exists(META_LIMIT, $this->metaPayload)) ? intval($this->metaPayload[META_LIMIT]) : $metaLimit;
        // test to ensure that the meta-limit passed does not exceed the limit unless the override is set
        if ($this->limitOverride and $limit > $metaLimit) $limit = $metaLimit;

        // build the options array for the query
        if ($skip) {
            $options[STRING_SKIP] = $skip;
            $this->strQuery .= '.skip(' . $skip . ')';
        }
        if ($limit) {
            $options[STRING_LIMIT] = $limit;
            $this->strQuery .= '.limit(' . $limit . ')';
        }

        // store readable info about the event
        $this->event = DB_EVENT_FETCH;
        $this->strQuery = DB_EVENT_FETCH . COLON . json_encode($query) . COMMA . json_encode($projection);

        try {
            // get the count of active records in the collection
            $this->recordsInCollection = $this->getQueryCount([ (DB_STATUS . $this->ext) => STATUS_ACTIVE ]);
            $this->recordsInQuery = $this->getQueryCount($query);
            if ($this->recordsInCollection == -1) {
                $this->eventMessages[] = ERROR_MDB_FETCH_COUNT_FAIL;
                $this->logger->warn(ERROR_MDB_FETCH_COUNT_FAIL);
                return;
            }

            // create the query object
            $queryObject = new MongoDB\Driver\Query($query, $options);

            // reset the data property
            $this->data = [];
            $this->count = 0;
            $this->recordsReturned = 0;

            // start the query timer
            if ($this->useTimers) $startTime = gasStatic::doingTime();

            // get the cursor object from the query execution and populate the $data property
            /** @var Cursor $cursor */
            $cursor = $this->manager->executeQuery($this->nameSpace, $queryObject, $this->readPreference);
            $cursor->setTypeMap(['root' => 'array', 'document' => 'array', 'array' => 'array']);

            // stop the query timer
            if ($this->useTimers) $this->doLogQueryTimer($this->strQuery, gasStatic::doingTime($startTime));
            $this->data = $cursor->toArray();           // gets all data, as array, in one line instead of looping
            $this->count = (empty($this->data)) ? 0 : count($this->data);
            $this->recordsReturned = $this->count;
            $this->state = STATE_SUCCESS;
            $this->status = true;
            $this->event = DB_EVENT_FETCH;
            // if the query returned no (not found) data
            if (0 == $this->count) {
                $this->state = STATE_NOT_FOUND;
                $this->eventMessages[] = INFO_QUERY_RETURNED_NO_DATA;
                $this->eventMessages[] = $this->strQuery;
            } else {
                // check for auditing and, if enabled for read events, publish an audit event to admin broker
                if ($this->useAuditing == AUDIT_NONDESTRUCTIVE) {
                    $this->auditRecordList = $this->data;
                    if (!$this->registerAuditEvent(EVENT_NAME_AUDIT_FETCH)) {
                        $this->eventMessages[] = ERROR_AUDIT_GENERIC_FAIL;
                        consoleLog($this->res, CON_ERROR, ERROR_AUDIT_GENERIC_FAIL);
                    }
                }
                // check to see if the filterData bypass was set
                if (isset($this->metaPayload[META_DONUT_FILTER]) and $this->metaPayload[META_DONUT_FILTER] == 1 and $this->metaPayload[META_CLIENT] == CLIENT_SYSTEM) return;
                // filter the return data set and cache if caching is enabled
                if (!$this->returnFilteredData())
                    $this->eventMessages[] = ERROR_DATA_PROCESSING;
            }
        } catch (MongoDB\Driver\Exception\InvalidArgumentException  |
                 MongoDB\Driver\Exception\ConnectionException       |
                 MongoDB\Driver\Exception\AuthenticationException   |
                 MongoDB\Driver\Exception\RuntimeException          |
                 Throwable $e)                                      {
            $hdr = sprintf(INFO_LOC, basename(__METHOD__), __LINE__);
            $this->eventMessages[] = ERROR_MONGO_EXCEPTION;
            @handleExceptionMessaging($hdr, $e->getMessage(), $foo, true);
            $this->state = STATE_DB_ERROR;
        }
    }


    /**
     * getQueryCount() -- public method
     *
     * todo - this is bad code and I should feel bad... write a wrapper for this method and make it private
     *
     * this method has one required input parameter - the query body (where discriminant) for the query.
     * this method has one options input parameter -- the string containing the name of the targeted collection.
     *
     * the method executes the query-count as a mongo command (as opposed to a query) and returns the the number of
     * records that match the query discriminant.
     *
     * the method returns an integer which is the record count.  if the method returns a -1, then an exception was
     * raised and trapped and the calling client should take the appropriate action.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_query
     * @param string $_cn -- collection name
     * @return int
     *
     *
     * HISTORY:
     * ========
     * 04-27-18     mks     _INF-188: original coding
     *
     */
    public function getQueryCount(array $_query, $_cn = ''): int
    {
        // get the collection count with a command:
        $cmd = [
            MONGO_STRING_COUNT =>  ((!empty($_cn)) ? $_cn : $this->collectionName),
            MONGO_STRING_QUERY => $_query
        ];
        try {
            $command = new MongoDB\Driver\Command($cmd);
            /** @var Cursor $result */
            $result = $this->manager->executeCommand($this->dbName, $command);
            $res = current($result->toArray());
            return ($res->n);
        } catch (MongoDB\Driver\Exception\InvalidArgumentException |
                 MongoDB\Driver\Exception\ConnectionException |
                 MongoDB\Driver\Exception\AuthenticationException |
                 MongoDB\Driver\Exception\RuntimeException |
                 Throwable $e) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_MONGO_EXCEPTION . $e->getMessage();
            $this->logger->warn($hdr . $msg);
            $this->eventMessages[] = $msg;
            $this->state = STATE_DB_ERROR;
            consoleLog($this->res, CON_SYSTEM, $hdr . $msg);
            return -1;
        }
    }


    /**
     * _updateRecord -- public method
     *
     * this is the public-facing entry point for updates to a collection - usually being invoked from the write broker.
     *
     * there are two input parameters to the method:
     *
     * $_data -- this is an aggregation of three associative arrays which contain the following two sub-arrays:
     *        QUERY_DATA -- the query data, similar to what's described for a fetch operation - this data will be
     *                      passed to the query builder for validation.  As such, the search criteria must adhere to
     *                      all the usual rules: indexed fields, correct data types, etc.
     *       UPDATE_DATA -- this is an associative array of cacheMapped key-value pairs representing the columns to
     *                      to be updated with the data for the update.
     *           OPTIONS -- the optional options array contains directives specific to the update event:
     *                      MULTI => Boolean:   if false, only the first matching document found in the query will
     *                                          be updated.  If true, all matching documents will be updated.  The
     *                                          default value for this parameter if not supplied is FALSE.
     *                      UPSERT => Boolean:  if false, the default value, then if the query finds no records, no
     *                                          data will be modified.  If the value is set to true and the query
     *                                          returns no records, then a new record will be inserted.
     *
     * Processing is straight-forward:
     *
     * 1.  validate the inputs
     * 2.  submit the query to the queryBuilder which will validate indexes and process for cacheMapping
     * 3.  validate the update data:  cacheMapping aliases, valid fields, valid data types.
     * 4.  update the data
     * 5.  return the results to the calling client via the state/status properties and the number of records
     *     successfully updated and, via eventMessages property, a list of fields that were dropped, if any.
     * 6.  if records were updated, and caching for the class is enabled, get a list of the affected records and
     *     post the updates to cache.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     *
     * HISTORY:
     * ========
     * 07-26-17     mks     CORE-464: original coding
     * 09-11-17     mks     CORE-558: protected fields functionality
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     * 02-15-19     mks     DB-116:   deprecated cache-mapping; is now handled at broker-services level, pre-query
     *                                fetch no longer dependent on journaling, cleaning records from cache post-update,
     *                                also option for multi is now being handled here based off the pre-query count
     * 06-21-19     mks     DB-128:   support for accurately handling META_LIMIT for mongoDB update
     * 10-25-19     mks     DB-136:   better error messaging by adding error location to output
     * 04-29-20     mks     ECI-132:  fixed error in evaluating not-found on pre-query fetch
     * 10-02-20     mks     DB-168:   improved error checking and handling - injected bypass on update protection if
     *                                the requesting client is not CLIENT_SYSTEM (admin)
     *
     */
    public function _updateRecord(array $_data):void
    {
        $validOperands = [ OPERAND_AND, OPERAND_NULL ];
        $otherOperands = [ OPERAND_OR, OPERAND_NOR ];
        $this->state = STATE_DATA_ERROR;
        $this->status = false;
        $method = basename(__METHOD__);
        $startTime = floatval(0);
        $options = (isset($_data[STRING_QUERY_OPTIONS])) ? $_data[STRING_QUERY_OPTIONS] : null;

        // check if updates are allowed for the current class
        if (!$this->allowUpdates) {
            $refuseUpdate = (isset($this->metaPayload[META_CLIENT]) and $this->metaPayload[META_CLIENT] == CLIENT_SYSTEM) ? true : false;
            if (!$refuseUpdate) {
                $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                $msg = ERROR_UPDATES_BY_CLASS_DENIED;
                $this->logger->data($hdr . $msg);
                $this->eventMessages[] = $hdr . $msg;
                return;
            }
        }

        // validate $_data as non-empty array
        if (empty($_data) or !is_array($_data)) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            $msg = ERROR_DATA_404;
            $this->logger->data($hdr . $msg);
            $this->eventMessages[] = $msg;
            return;
        }

        // validate that the search criteria exists
        if (empty($_data[STRING_QUERY_DATA])) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            $msg = ERROR_PARAM_404 . STRING_QUERY_DATA;
            $this->logger->data($hdr . $msg);
            $this->eventMessages[] = $msg;
            return;
        }

        // validate the update data and inject the status field if necessary
        if (empty($_data[STRING_UPDATE_DATA]) or !is_array($_data[STRING_UPDATE_DATA])) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            $msg = ERROR_PARAM_404 . STRING_UPDATE_DATA;
            $this->logger->data($hdr . $msg);
            $this->eventMessages[] = $msg;
            return;
        } else {
            // with the new cacheMapping, we need to add extensions to the updateData payload
            $this->updateBuilder($_data[STRING_UPDATE_DATA]);
            if (is_null($_data[STRING_UPDATE_DATA])) {
                $hdr = sprintf(INFO_LOC, $method, __LINE__);
                @handleExceptionMessaging($hdr, ERROR_UPDATE_DATA_INVALID, $this->eventMessages, true);
                if (!empty($_data[STRING_UPDATE_DATA]))
                    $this->logger->data(json_encode($_data[STRING_UPDATE_DATA]));
                else
                    $this->logger->data(STRING_UPDATE_DATA . COLON . STRING_NOT_DEFINED);
                $this->status = false;
                $this->state = STATE_VALIDATION_ERROR;
                return;
            }
            $updateString = json_encode($_data[STRING_UPDATE_DATA]);
            $updateData = [ MONGO_SET => $_data[STRING_UPDATE_DATA]];
        }

        // DB-88: if a class supports soft-deletes, inject the STATUS discriminant (!= DELETED) in the query
        //        Follow-up:  since payload validation happens at the broker level, before we get this deep,
        //                    all array keys should be the fully-qualified schema column names so there's no
        //                    need to check on extension-less or cache-mapped values...
        $queryData = $_data[STRING_QUERY_DATA];
//        if (!$this->useDeletes and (!in_array(DB_STATUS, $_data[STRING_QUERY_DATA]) or !in_array(CM_TST_FIELD_TEST_STATUS, $_data[STRING_QUERY_DATA]))) {
        if (!$this->useDeletes and (!array_key_exists((DB_STATUS . $this->ext), $queryData))) {
            $newQueryData = [];
            foreach ($queryData as $queryKey => $queryValue) {
                if (in_array($queryKey, $validOperands)) {
                    $newQueryData[DB_STATUS] = [ OPERAND_NULL => [ OPERATOR_DNE => [ STATUS_DELETED ]]];
                    if ($queryKey == OPERAND_NULL)
                        $newQueryData[] = [ OPERAND_AND => null ];
                    else
                        $newQueryData[$queryKey] = $queryValue;
                } elseif (in_array($queryKey, $otherOperands)) {
                    $newQueryData[$queryKey] = $queryValue;
                    $newQueryData[DB_STATUS] = [ OPERAND_NULL => [ OPERATOR_DNE => [ STATUS_DELETED ]]];
                    $newQueryData[] = [ OPERAND_AND => null ];
                } else {
                    $newQueryData[$queryKey] = $queryValue;
                }
            }
            $queryData = $newQueryData;
        }

        // the query to the query builder and see if it builds  (qb appends the class extension to fields, fyi...)
        $originalQuery = $this->queryBuilder($queryData);
        if (is_null($originalQuery) or $this->status === false) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            $msg = ERROR_DATA_QUERY_BUILD . COLON . json_encode($queryData);
            @handleExceptionMessaging($hdr, $msg, $this->eventMessages, true);
            return;
        }

        // after checking for protected fields, there is a weird fringe case that, if all of the fields to be updated
        // were protected, and we've removed those fields, then the update payload would be empty.  Need to test for
        // this and, if empty, terminate the request.  (THIS SHOULD NOT EXECUTE!)
        if (empty($updateData['$set']) or count($updateData['$set']) == 0) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            @handleExceptionMessaging($hdr, ERROR_UPDATE_PAYLOAD_EMPTY_POST_VALIDATION, $this->eventMessages, true);
            $this->status = false;
            $this->state = STATE_VALIDATION_ERROR;      // must be validation error
            return;
        }

        // wrap the mongo events in an exception handler
        try {
            // get a list of the records impacted by the update so that we can remove them from cache post-update
            // records are stored in $auditRecordList
            if (!$this->fetchRecordsBeforeChange($originalQuery)) {
                // query was successful but no records were returned
                if ($this->state == STATE_NOT_FOUND) {
                    $hdr = sprintf(INFO_LOC, $method, __LINE__);
                    $this->eventMessages[] = INFO_QUERY_RETURNED_NO_DATA;
                    $this->logger->data($hdr . INFO_QUERY_RETURNED_NO_DATA);
                    $this->status = true;
                    return;
                }
                // query failed for another reason, Namaste was unable to exec the pre-query fetch
                $hdr = sprintf(INFO_LOC, $method, __LINE__);
                $msg = ERROR_AUDIT_REC_LIST;
                $this->eventMessages[] = $msg;
                if ($this->debug) $this->logger->debug($hdr . $msg);
                $this->status = false;
                $this->state = STATE_DATA_ERROR;
            }
            // check the count of the records to be fetched and if > 1, set the STRING_MULTI option
            if (!isset($options[STRING_MULTI]) and count($this->recordTokenList) > 1) $options[STRING_MULTI] = true;
            // init the bulkWrite class object and send the request off for execution
            $fatso = new MongoDB\Driver\BulkWrite();   // unordered
            $newQuery = [ ($this->pKey . $this->ext) => [ "\$in" => $this->recordTokenList ]];
            $fatso->update($newQuery, $updateData, $options);
//            $fatso->update($originalQuery, $updateData, $options);
            $this->strQuery = 'update(' . json_encode($newQuery) . ')' . COMMA . 'u: (' . $updateString . ')';
            $this->doBulkWrite($fatso);

            // if we successfully exec'd the update, and we have caching enabled, we're going to need to pull the
            // affected records and store those into cache, then build a list of cache-keys.
            if (!$this->status) {
                $hdr = sprintf(INFO_LOC, $method, __LINE__);
                $msg = ERROR_NOSQL_UPDATE;
                $this->eventMessages[] = $msg;
                $this->logger->error($hdr . $msg);
                $this->state = STATE_DB_ERROR;
                return;
            }
            // reset the $data and associated properties
            $this->data = array();
            $this->count = 0;

            // build the query object to fetch the updated records for return
            // start the query timer?
            if ($this->useTimers) $startTime = gasStatic::doingTime();
            $query = new MongoDB\Driver\Query($newQuery);
            // execute the query and return the updated records
            /** @var Cursor $cursor */
            $cursor = $this->manager->executeQuery($this->nameSpace, $query);
            $cursor->setTypeMap(['root' => 'array', 'document' => 'array', 'array' => 'array']);
            // stop the query timer?
            if ($this->useTimers) $this->doLogQueryTimer($this->strQuery, gasStatic::doingTime($startTime));
            $this->data = $cursor->toArray();           // gets all data, as array, in one line instead of looping
            $this->count = (!empty($this->data)) ? count($this->data) : 0;
            $this->status = true;
            $this->state = ($this->count == 0) ? STATE_NOT_FOUND : STATE_SUCCESS;
            $this->event = DB_EVENT_UPDATE;
            // if auditing is enabled, copy of the original records and invoke the audit/journal processing
            if ($this->useAuditing > AUDIT_NOT_ENABLED) {
                if (!$this->registerAuditEvent(EVENT_NAME_AUDIT_UPDATE)) {
                    $this->eventMessages[] = ERROR_AUDIT_GENERIC_FAIL;
                    consoleLog($this->res, CON_ERROR, ERROR_AUDIT_GENERIC_FAIL);
                }
            }
            // return the filtered data set
            if (!$this->returnFilteredData())
                $this->eventMessages[] = ERROR_DATA_PROCESSING;
            // clear updated records from cache
            if (!empty($this->auditRecordList)) {
                if (!gasCache::smashCache($this->recordTokenList, $this->eventMessages))
                    $this->logger->warn(ERROR_CACHE_SMASH_FAIL_SYSTEM . $this->strQuery);
            }
        } catch(MongoDB\Driver\Exception\InvalidArgumentException |
                MongoDB\Driver\Exception\ConnectionException |
                MongoDB\Driver\Exception\RuntimeException |
                Throwable | TypeError $t) {
            $hdr = sprintf(INFO_LOC, $method, __LINE__);
            @handleExceptionMessaging($hdr, $t->getMessage(), $foo, true);
            $this->state = STATE_DB_ERROR;
            return;
        }
    }


    /**
     * deleteRecord() -- public method
     *
     * this is the delete method used to remove records from a collection.
     *
     * There is one input parameter to the method.  As with all CRUD key methods, the instantiation of the class
     * loads the broker-event meta data payload into the class.
     *
     * The content of $_data should be a query array, one that follows the rules for a mongo query in that it can
     * be successfully processed by the query builder and becomes, in turn, the discriminant for removing collection
     * records.
     *
     * because the query-builder validates the data (fields are known, data types match, cache-mapping, etc.) a
     * valid return from the query build green-lights us to proceed with the delete operation.
     *
     * One further consideration will be taken into account - whether or not the class supports hard or soft
     * deletes.  If the former, the records will be removed.  If soft-deletes are supported, we'll make a sideways
     * call to _updateRecord passing in the original query data and a directive to update the status field to
     * a deleted status.  tl;dr:  this method only handles hard deletes.  Soft deletes are passed-off to the
     * update method.
     *
     * Last, if we successfully completed the delete request, and if cache is enabled for the class, we'll update
     * the cache store by removing any of the referenced records.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     *
     * HISTORY:
     * ========
     * 07-19-17     mks     CORE-464: original coding
     * 08-07-17     mks     CORE-497: update cache on delete event
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     * 11-02-18     mks     DB-70: updated for audit/journaling
     * 01-09-19     mks     DB-80: fixed bug in creating delete query by removing the class extension
     * 02-15-19     mks     DB-116: deprecated cache update; is not handled at broker-services level,
     *                              cleaned-up exception handling, added tokenList population
     * 06-21-19     mks     DB-128: support for META_LIMIT for mongoDB DELETE
     *
     */
    public function _deleteRecord(array $_data):void
    {
        $this->state = STATE_DATA_ERROR;
        $this->status = false;

        if (empty($_data) or !is_array($_data)) {
            $msg = ERROR_PARAM_404 . STRING_DATA;
            $this->eventMessages[] = $msg;
            $this->logger->data($msg);
            return;
        }
        // do the soft-delete status-injection thing
        $query = $_data[STRING_QUERY_DATA];
        $oldQuery = $query;
        list($query, $injection) = $this->softDeleteStatusInjection($query);

        // test the query -- no point in going further if it doesn't build
        $query = $this->queryBuilder($query);
        if (!$this->status) {
            if (!$injection) {
                // we didn't inject a query so log the failure and return
                $this->eventMessages[] = ERROR_DATA_QUERY_BUILD;
                $this->logger->error(ERROR_DATA_QUERY_BUILD . COLON . $this->collectionName . COLON . json_encode($query));
                return;
            } elseif (!empty($oldQuery)) {
                // attempt to build the original query
                $query = $this->queryBuilder($oldQuery);
                if (!$this->status) {
                    $this->eventMessages[] = ERROR_DATA_QUERY_BUILD;
                    $this->logger->error(ERROR_DATA_QUERY_BUILD . COLON . $this->collectionName . COLON . json_encode($oldQuery));
                    return;
                }
            }
        }

        // first step - get a list of records that will be impacted (deleted) by the query
        if (!$this->fetchRecordsBeforeChange($query)) {
            $this->eventMessages[] = ERROR_NOSQL_FETCH;
            return;
        }
        if (count($this->auditRecordList) == 0) {
            $this->eventMessages[] = INFO_QUERY_RETURNED_NO_DATA;
            consoleLog($this->res, CON_SYSTEM, INFO_QUERY_RETURNED_NO_DATA);
            return;
        }

        // copy the record token list over to a legacy member variable for auditing
        if ($this->useAuditing > AUDIT_NOT_ENABLED) $this->tokenList[] = $this->recordTokenList;

        try {
            // check to see if this is a soft-delete class, and if so, pass the work off to the update method
            if ($this->useDeletes === false) {
                $_data[STRING_UPDATE_DATA] = [DB_STATUS => STATUS_DELETED];
                $_data[STRING_QUERY_OPTIONS] = (count($this->auditRecordList) > 1) ? [STRING_UPSERT => false] : $_data[STRING_QUERY_OPTIONS] = [STRING_MULTI => true, STRING_UPSERT => false];
                $oldAuditSetting = $this->useAuditing;
                $oldJournalSetting = $this->useJournaling;
                // you have to unset the audit/journal options for the delete because the delete is calling
                // update and that will trigger a duplicate audit...
                $this->useAuditing = false;
                $this->useJournaling = false;
                $this->_updateRecord($_data);
                $this->useAuditing = $oldAuditSetting;
                $this->useJournaling = $oldJournalSetting;
                $strQuery = [
                    BROKER_REQUEST => BROKER_REQUEST_UPDATE,
                    BROKER_DATA => $_data,
                    BROKER_META_DATA => $this->metaPayload
                ];
                $this->strQuery = json_encode($strQuery);
                if ($this->useAuditing) @$this->registerAuditEvent(EVENT_NAME_AUDIT_DELETE);
            } else {
                // we need to re-factor the delete query s.t. the discriminant is based on the token list
                // is the one we'll use to delete the records instead of the query submitted by the client
                $fatso = new MongoDB\Driver\BulkWrite();
                $newQuery = [ (DB_TOKEN . $this->ext) => [ "\$in" => $this->recordTokenList ]];
                $fatso->delete($newQuery, [STRING_LIMIT => 0]);  // delete all documents that match the query predicate
                $this->doBulkWrite($fatso);
                $this->strQuery = json_encode($newQuery);
                if ($this->useAuditing) @$this->registerAuditEvent(EVENT_NAME_AUDIT_DELETE);
            }
            // spin through the tokenList and remove all these records from cache
            if (!gasCache::smashCache($this->recordTokenList, $this->eventMessages))
                $this->logger->warn(ERROR_CACHE_SMASH_FAIL_SYSTEM . $this->strQuery);
        } catch (MongoDB\Driver\Exception\InvalidArgumentException | Throwable | TypeError $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->state = STATE_FRAMEWORK_WARNING;
        }
    }


    /**
     * _checkQuery() -- public function
     *
     * this function is mandated by the Namaste core abstraction.  The method provides access to the query-builder
     * functionality within the the data classes to validate a query submission from clients.
     *
     * The method requires a single input parameter:  this is the array containing the query to be tested.
     *
     * The method returns a boolean indicating whether or not the query was successfully built... if it was built
     * successfully, then store the query in the class member container.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_query
     * @return bool
     *
     *
     * HISTORY:
     * ========
     * 04-23-18     mks     _INF-188: Original coding
     *
     */
    public function _checkQuery(array $_query): bool
    {
        try {
            $query = $this->queryBuilder($_query);
            if ($this->status) $this->strQuery = $query;
            return $this->status;
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->state = STATE_FRAMEWORK_WARNING;
            return false;
        }
    }


    /**
     * getDBName() - public method
     *
     * This is a simple, public-access method, for fetching the name of the current class' database name.  As of this
     * writing, this method is used in the mongoConfig.php script for generating collection indexes.
     *
     * This method leverages the getCollectionName() method by passing a parameter to that method that tells the
     * method we want the collection name returned from the nameSpace property of the current instantiation.
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @return null|string
     *
     * HISTORY:
     * ========
     * 08-11-17     mks     CORE-467: original coding
     *
     */
    public function getDBName(): ?string
    {
        return ($this->getCollectionName(0));
    }


    /**
     * getCollectionName() -- public method
     *
     * This method accepts one (optional) input parameter that specified which part of the currently defined
     * mongo name-space we wish returned.  If no input is specified, it defaults to the value of (1) which will
     * return the collection name.  If a 0 is passed, then the database name will be returned.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param int $_which  - use 0 to return database name, 1 or null to return the collection name
     * @return null|string
     *
     * HISTORY:
     * ========
     * 08-11-17     mks     CORE-467: original coding
     *
     */
    public function getCollectionName($_which = 1): ?string
    {
        return(isset($this->nameSpace) ? ((false === ($ns = preg_split('/[.]/', $this->nameSpace))) ? null : $ns[$_which]) : null);
    }


    /**
     * pushSubCollectionEvent() -- public method
     *
     * this method is generic and is used to add sub-array-collection elements. Historically, this method has been used
     * to manage record histories.  However, the method is being made available so that we can provide the versatility
     * of being able to push/pop sub-collection records as needed.  (And without adult supervision!)
     *
     * As such, it should be understood by the client that sub-collection management of data is limited to just the
     * sub-collection field of a record.  Users wanting to update multiple fields within a record must make two calls
     * if one of those updates is twiddling with the sub-collection records.
     *
     * todo: document sub-collection processing requirements
     *
     * the method has one, required, input parameter: $_data which contains the following indexes:
     *
     * STRING_GUID_KEY:
     * the primary key field identifying which tuple in the collection that will be updated
     * STRING_SUBC_FIELD
     * the sub-collection name under which $_data will be added
     * STRING_DATA
     * the sub-collection event data (an array of records)
     *
     * Note the following requirements:
     *   -- the $_field string must be a valid member of the the current fieldList
     *   -- $_data cannot be empty or a non-array
     *   -- $_data must be an array of records even if there exists only 1 record
     *   -- $_pk must be a GUID and a record with that GUID must exist
     *
     * Post-validation, we'll build a simple mongo update query using the $push directive which will insert the
     * new sub-array tuple ($_data) into the array named by $_field.
     *
     * The request success is determined by the class state/status variables and is the responsibility of the calling
     * client to evaluate.
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param $_data
     *
     * HISTORY:
     * ========
     * 08-24-17     mks     CORE-494: initial coding
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     * 03-28-19     mks     DB-116:   check that number of sub-collection records does not exceed the max-record limit,
     *                                added metrics (query timer) to the dbWrite event
     *
     */
    public function pushSubCollectionEvent(array $_data): void
    {
        $this->state = STATE_VALIDATION_ERROR;
        $this->status = false;
        $msg = '';
        $startTime = floatval(0);
        $requiredFields = [ STRING_GUID_KEY, STRING_SUBC_FIELD, STRING_DATA ];

        // extract the required fields from the parameter: $_data
        foreach ($requiredFields as $requiredField) {
            if (!array_key_exists($requiredField, $_data)) {
                $msg = ERROR_DATA_KEY_404 . $requiredField;
                $this->eventMessages[] = $msg;
            }
        }
        if (!empty($msg)) return;
        $pk = $_data[STRING_GUID_KEY];
        $field = $_data[STRING_SUBC_FIELD];
        $data = $_data[STRING_DATA];

        // if only one record is submitted and it's not embedded in an indexed array, embed it
        if (array_key_first($data) !== 0 and is_array($data)) {
            $data = [$data];
        } else {
            // else, ensure that we're not inserting more than the max-allowable number of records
            $recordLimit = gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT];
            if (count($data) > $recordLimit) {
                $this->state = STATE_DATA_ERROR;
                $this->eventMessages[] = ERROR_RECORD_LIMIT_EXCEEDED . strval($recordLimit);
                return;
            }
        }
        // yes, we previously validated during cacheMapping - but this inserts tokens into each subC record
        try {
            if (!$this->validateSubCollectionData($pk, $field, $data)) return;
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->state = STATE_FRAMEWORK_WARNING;
            return;
        }
        // if we make it to this point, the input params are validated - do the pre-fetch
//        $query = [ STRING_QUERY_DATA => [(DB_TOKEN . $this->ext) => [ OPERAND_NULL => [ OPERATOR_EQ => [ $pk ]]]]];
//        $clone = clone $this;
//        $clone->_fetchRecords($query);
//        if ($clone->status) {
//            $ogChecksum = md5(gzcompress(json_encode($clone->getData())));
//            // todo (ECI-137) -- use $clone as your foundation for creating audit/journal records which you don't have!
//        } else {
//            if (is_object($clone)) $clone->__destruct();
//            unset($clone);
//            $this->state = STATE_FRAMEWORK_WARNING;
//            $this->logger->warn(ERROR_CLONE_QUERY . json_encode($query));
//            $this->eventMessages[] = INFO_GENERIC_DB_ERROR;
//            return;
//        }

        // build the query data
        $query = [ (DB_TOKEN . $this->ext) => $pk ];
        $strQuery = json_encode($query);

        $modifier = [ MONGO_PUSH => [ $field => $data ]];
        $strModifier = json_encode($modifier);

        // init the bulkWrite class object - we have to iterate through the data and add each modifier separately
        // to the update() command.
        try {
            if ($this->useTimers) $startTime = gasStatic::doingTime();
            $fatso = new MongoDB\Driver\BulkWrite();   // unordered
            foreach ($data as $record) {
                $modifier = [ MONGO_PUSH => [ $field => $record]];
                $fatso->update($query, $modifier);
            }
            if ($this->useTimers) $this->doLogQueryTimer($this->strQuery, gasStatic::doingTime($startTime));
        } catch (MongoDB\Driver\Exception\InvalidArgumentException | Throwable $e) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_MONGO_EXCEPTION . $e->getMessage();
            $this->eventMessages[] = $msg;
            $this->logger->warn($hdr . $msg);
            consoleLog($this->res, CON_SYSTEM, $hdr . $msg);
            $this->state = STATE_DB_ERROR;
            return;
        }
        $this->doBulkWrite($fatso);

        // if the request was successfully written to the db - fetch the updated record which also updates cache
        if ($this->status) {
            $query = [ STRING_QUERY_DATA => [(DB_TOKEN . $this->ext) => [ OPERAND_NULL => [ OPERATOR_EQ => [ $pk ]]]]];
            try {
                $this->_fetchRecords($query);
            } catch (Throwable $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
                $this->eventMessages[] = $msg;
                if (isset($this->logger) and $this->logger->available)
                    $this->logger->error($hdr . $msg);
                else
                    consoleLog($this->res, CON_ERROR, $hdr . $msg);
                $this->state = STATE_FRAMEWORK_WARNING;
                return;
            }
        }
        $this->strQuery = 'subC_update(' . $strQuery . ')' . COMMA . '(' . $strModifier . ')';
    }


    /**
     * popSubCollection() -- public function
     *
     * this method is the broker-access point for removing a sub-collection record from a sub-collection.
     *
     * there is one input parameter for the method - an associative array of data containing the following
     * parameters as the keys to the array:
     *
     * STRING_GUID_KEY   -- references which record in the collection to update
     * STRING_SUBC_FIELD -- the name of the subCollection (root level)
     * STRING_SUBC_GUID  -- references the sub-collection key for the search criteria
     *
     * first steps is to validate the input data, importantly, check the value for the record guid by querying
     * the class for the record.  Next, we'll validate that the sub-collection (container) field is valid, then
     * the sub-collection data field exists.
     *
     * In all cases, the name of the sub-collection qualifying field should be the GUID assigned to the sub-collection
     * record -- this means that the client user must have pre-knowledge (or a reason) to remove a sub-collection
     * record.  (You can't arbitrarily nuke sub-c records.)
     *
     * todo: CORE-555: expand sub-collection processing to delete based on a proper query range or multiple guids
     * todo: sub-collection deletes needs to be covered under audit and journaling
     *
     * Once data validation is complete, build the query and submit it to mongo for processing.
     *
     * Note that sub-collection record removal isn't subjected to hard or soft-delete states.  All the records
     * specified are hard-deleted.
     *
     * also note that this method, unlike the sub-collection push method (above), can only handle one sub-collection
     * record event at-a-time.
     *
     * errors in validation or processing will cause execution to return immediately to the calling client.  Success
     * can be derived by the class state/status member variables, the diagnostics array container, and the log files.
     *
     * on success, we'll update the record in-cache and return the cache-key to the client as payload data if caching
     * is enabled on the class -- otherwise return the record as a normal query.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data -- an associative array containing the query elements
     *
     *
     * HISTORY:
     * ========
     * 09-05-17     mks     CORE-494: original coding
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     * 03-28-19     mks     DB-116:   refactored for broker-level cache-mapping, removed some redundant validation
     *                                checks, streamlined exception handling, added metrics query timers
     *
     */
    public function popSubCollection(array $_data): void
    {
        $this->state = STATE_VALIDATION_ERROR;
        $this->status = false;
        /** @noinspection PhpUnusedLocalVariableInspection */
        $foundError = false;
        $startTime = floatval(0);

        // class has to support subCollections
        if (!isset($this->subCollections) or empty($this->subCollections)) {
            $this->logger->debug(ERROR_SUBC_404);
            $this->eventMessages[] = ERROR_SUBC_404;
            return;
        }

        // does the data payload have data?
        if (empty($_data) or !is_array($_data)) {
            $this->eventMessages[] = ERROR_DATA_404;
            $this->logger->debug(ERROR_DATA_404);
            return;
        }

        // does the data payload have the right data?
        $foundError = false;
        $validFields = [ STRING_GUID_KEY, STRING_SUBC_FIELD, STRING_SUBC_GUID ];
        foreach ($validFields as $field) {
            if (!array_key_exists($field, $_data)) {
                $msg = ERROR_DATA_KEY_404 . $field;
                $this->eventMessages[] = $msg;
                $this->logger->debug($msg);
                $foundError = true;
            }
            if ($foundError) return;
        }

        // break-out the payload data into it's respective variables
        $recordGUID = $_data[STRING_GUID_KEY];
        $recordSubCollectionField = $_data[STRING_SUBC_FIELD] . $this->ext;
        $subCollectionRecordGUID = $_data[STRING_SUBC_GUID];

        if ($this->useCache and (!array_key_exists($recordSubCollectionField, $this->cacheMap))) {
            $foundError = true;
        } elseif (!in_array($recordSubCollectionField, $this->fieldList)) {
            $foundError = true;
        }

        if ($foundError) {
            $msg = ERROR_DATA_FIELD_NOT_MEMBER . STRING_SUBC_FIELD;
            $this->eventMessages[] = $msg;
            $this->logger->debug($msg);
            return;
        }

        try {
            // build the query
            $query = [ (DB_TOKEN . $this->ext) => $recordGUID ];
            $modifier = [
                MONGO_PULL => [ $recordSubCollectionField => [ (DB_TOKEN . $this->ext) => $subCollectionRecordGUID ] ]
            ];
            $this->strQuery = json_encode($query) . COLON . json_encode($modifier);

            // init the bulkWrite class object - we have to iterate through the data and add each modifier separately
            // to the update() command.
            if ($this->useTimers) $startTime = gasStatic::doingTime();
            $gordo = new MongoDB\Driver\BulkWrite();   // unordered
            $gordo->update($query, $modifier);
            // write (delete) the sub-collection record
            $this->doBulkWrite($gordo);
            if ($this->useTimers) $this->doLogQueryTimer($this->strQuery, gasStatic::doingTime($startTime));
            // check the return and that we did the update
            $oldState = null;
            if ($this->status) {
                if ($this->bwResult->getMatchedCount() > 0) {
                    if ($this->bwResult->getModifiedCount() == 0) {
                        // sub-collection record was not found
                        $oldState = STATE_NOT_FOUND;
                        $this->eventMessages[] = ERROR_SUBC_RECORD_404 . $subCollectionRecordGUID;
                    } elseif ($this->bwResult->getModifiedCount() == 1) {
                        // sub-collection record successfully updated
                        $this->eventMessages[] = sprintf(SUCCESS_SUBC_RECORD_DELETED, $subCollectionRecordGUID);
                    }
                }
                // fetch or re-cache the updated record
                $fQuery = [STRING_QUERY_DATA => [(DB_TOKEN . $this->ext) => [OPERAND_NULL => [OPERATOR_EQ => [$recordGUID]]]]];
                $this->_fetchRecords($fQuery);
            }
        } catch (MongoDB\Driver\Exception\InvalidArgumentException | Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->state = STATE_FRAMEWORK_WARNING;
            return;
        }
        // save the original (delete) query
        $this->strQuery = json_encode($query) . COLON . json_encode($modifier);
        if (!is_null($oldState)) $this->state = $oldState;
    }


    /**
     * fetchSubCollectionRecord() -- public method
     *
     * This method allows a client to fetch a record by querying the sub-collection fields.  By definition, when we
     * call a sub-collection-fetch event, we're implicitly stating that you're going to limit your query to the
     * sub-collection columns only.
     *
     * In other words, you can't query a record using both a root level column and a column nested within a
     * sub-collection.  As of this writing, that functionality does not exist within Namaste.
     *
     * There's only one input parameter to the function and that's the data payload, after it's been validated and
     * processed by the broker.
     *
     * The function returns void - success in processing must be validated by evaluating member variables state and/or
     * status to determine if the method completed successfully.  Errors, if encountered, will be stored in the
     * $eventMessages container and will also be echo'd as debug log entries.
     *
     * The validation ensures that the required data fields are present, in the correct format, and populated.  Keys in
     * the STRING_SUBC_DATA container are validated against the sub-collection field names.
     *
     * Note that skip and limit (via Meta fields) are fully supported for this method.
     *
     * Next, we build the query which is predicated on a couple of checks - if the current class supports soft-deletes,
     * then we'll need to add a status ( <> DELETED ) discriminant to the query.  Second, if there is only one
     * sub-collection field to be queried, then we'll form a vanilla "find" query.  If more than one sub-collection
     * column is queried, we'll build the query using the mongo $elemMatch operator.
     *
     * Note that sub-collection queries containing multiple discriminants are joined using a logical AND operand.  There
     * is not support for OR in these queries, or any other operand other than AND.
     *
     * Note that you may only query one sub-collection column at a time.
     *
     * Note that auditing for sub-collection fetching is fully supported.
     *
     * If the query builds and executes successfully, we'll populate member data with the relevant information before
     * returning control to the calling client.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     *
     *
     * HISTORY:
     * ========
     * 04-10-20     mks     ECI-107: original coding completed
     * 04-21-20     mks     ECI-107: refactor for new data payload
     *
     */
    public function fetchSubCollectionRecord(array $_data):void
    {
        $this->state = STATE_VALIDATION_ERROR;
        $this->status = false;
        $hasErrors = false;
        $startTime = floatval(0);
        $queryData = [];

        // class has to support subCollections
        if (!isset($this->subCollections) or empty($this->subCollections)) {
            $this->logger->debug(ERROR_SUBC_404);
            $this->eventMessages[] = ERROR_SUBC_404;
            $hasErrors = true;
        }

        // does the data payload have data?
        if (empty($_data) or !is_array($_data)) {
            $this->eventMessages[] = ERROR_DATA_404;
            $this->logger->debug(ERROR_DATA_404);
            $hasErrors = true;
        }

        // does the data payload have the right data?
        if (!array_key_exists(STRING_SUBC_COL, $_data)) {
            $this->eventMessages[] = ERROR_DATA_KEY_404 . STRING_SUBC_FIELD;
            $this->logger->debug(ERROR_DATA_KEY_404 . STRING_SUBC_FIELD);
            $hasErrors = true;
        }
        if (!array_key_exists(STRING_SUBC_DATA, $_data)) {
            $this->eventMessages[] = ERROR_DATA_KEY_404 . STRING_SUBC_DATA;
            $this->logger->debug(ERROR_DATA_KEY_404 . STRING_SUBC_DATA);
            $hasErrors = true;
        }
        $subCColumn = $this->FQCN($_data[STRING_SUBC_COL]);
        if (is_null($subCColumn)) {
            $msg = sprintf(ERROR_ARRAY_KEY_UNK, $_data[STRING_SUBC_FIELD]);
            $this->eventMessages[] = $msg;
            $this->logger->debug($msg);
            $hasErrors = true;
        }
        // test to ensure that STRING_SUBC_COLUMN keys are members of $this->subCollections
        if (!array_key_exists($subCColumn, $this->subCollections)) {
            $this->eventMessages[] = sprintf(ERROR_SUBC_KEY_404, $_data[META_SUBC_FIELD]);
            $this->logger->debug(sprintf(ERROR_SUBC_KEY_404, $subCColumn));
            $hasErrors = true;
        }
        $qData = $_data[STRING_SUBC_DATA];
        // ensure that the subc-Data is an array...
        if (!is_array($qData)) {
            // ...if not, generate a diagnostic message
            $this->eventMessages[] = ERROR_DATA_ARRAY_NOT_ARRAY . STRING_SUBC_FIELD;
            $this->logger->debug(ERROR_DATA_ARRAY_NOT_ARRAY . STRING_SUBC_FIELD);
            $hasErrors = true;
        } else {
            // ... else validate the keys as sub-collection members
            foreach ($qData as $key => $value) {
                $tKey = $this->FQCN($key);
                if (is_null($tKey) or !in_array($tKey, $this->subCollections[$subCColumn])) {
                    $this->eventMessages[] = ERROR_SUB_COLLECTION_NOT_MEMBER . $key;
                    $this->logger->debug(ERROR_SUB_COLLECTION_NOT_MEMBER . $key);
                    $hasErrors = true;
                } else {
                    // check the submitted data type against the column's declared type
                    if ($this->fieldTypes[$tKey] != gettype($value)) {
                        $hdr = sprintf(INFO_LOC, basename(__METHOD__), __LINE__);
                        $msg = sprintf(ERROR_DATA_TYPE_MISMATCH_DETAILS, $tKey, $this->fieldTypes[$tKey], gettype($value));
                        $this->logger->error($hdr . $msg);
                        consoleLog($this->res, CON_ERROR, $hdr . $msg);
                        $this->eventMessages[] = ERROR_DATA_TYPE_MISMATCH . COLON . $key;
                        $hasErrors = true;
                    }
                    $queryData[$tKey] = $value;
                }
            }
        }   // NOTE: fully qualified query data is now stored in $queryData array

        // if there were validation or verification errors, then return now before processing starts
        if ($hasErrors) return;

        // process skip/limit values
        // first, check to see if the limit override has been set
        $metaLimit = intval(gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT]);
        $this->limitOverride = (array_key_exists(META_LIMIT_OVERRIDE, $this->metaPayload) and $this->metaPayload[META_LIMIT_OVERRIDE] === 1) ? true : false;
        $skip = (array_key_exists(META_SKIP, $this->metaPayload)) ? intval($this->metaPayload[META_SKIP]) : 0;
        // if limit was passed in-Meta, set to the meta value, else set to the system max
        $limit = (array_key_exists(META_LIMIT, $this->metaPayload)) ? intval($this->metaPayload[META_LIMIT]) : $metaLimit;
        // test to ensure that the meta-limit passed does not exceed the limit unless the override is set
        if ($this->limitOverride and $limit > $metaLimit) $limit = $metaLimit;

        // set-up the options array:
        // filter-out the _id field from the query return
        // add skip/limit values if set
        $options = [STRING_PROJECTION => [ MONGO_ID => 0]];
        if ($skip) $options[STRING_SKIP] = $skip;
        if ($limit) $options[STRING_LIMIT] = $limit;

        // if we get to this point, we have all the requisite data -- start to build the query...
        // if we have only one discriminant in the query, then we're going to build a (relatively) simple "find" query
        // if there are multiple discriminants, then we need to build an $elementMatch query using logical and
        if (count($queryData) == 1) {
            // build a find query
            $key = key($queryData);
            $val = $queryData[$key];
            if (!$this->useDeletes) {
                $query = [ MONGO_AND => [[ (DB_STATUS . $this->ext) => [ MONGO_DNE => STATUS_DELETED ], $subCColumn . DOT . $key => $val ]]];
            } else {
                $query = [ $key => $val ];
            }
        } else {
            // build the elemMatch query
            if (!$this->useDeletes) {
                // { $and : [{ companyContacts_api : { $elemMatch : { $and : [ { employeeName_api : "Micheal Shallop"}, {employeeEmail_api : "mike@goodgirl.jewelry"} ]}}}, {status_api: {$ne : "DELETED"}} ]}
                $query = [MONGO_AND => [[$subCColumn => [MONGO_ELEMENT_MATCH => [MONGO_AND => [$queryData]]]], [(DB_STATUS . $this->ext) => [MONGO_DNE => STATUS_DELETED]]]];
            } else {
                // { companyContacts_api : { $elemMatch : { $and : [ { employeeName_api : "Micheal Shallop"}, {employeeEmail_api : "mike@goodgirl.jewelry"} ]}}}
                $query = [$subCColumn => [MONGO_ELEMENT_MATCH => [MONGO_AND => [$queryData]]]];
            }
        }
        $this->event = DB_EVENT_SCF;
        $this->strQuery = json_encode($query);
        if ($skip) $this->strQuery .= '.skip(' . $skip . ')';
        if ($limit) $this->strQuery .= '.limit(' . $limit . ')';
        try {
            // reset the data property
            $this->data = [];
            $this->count = 0;
            $this->recordsReturned = 0;

            // instantiate a mongoDB query object
            $queryObject = new MongoDB\Driver\Query($query, $options);

            // start the query timer
            if ($this->useTimers) $startTime = gasStatic::doingTime();                // create the query object

            // get the cursor object from the query execution and populate the $data property
            /** @var Cursor $cursor */
            $cursor = $this->manager->executeQuery($this->nameSpace, $queryObject, $this->readPreference);
            $cursor->setTypeMap(['root' => 'array', 'document' => 'array', 'array' => 'array']);

            // stop the query timer
            if ($this->useTimers) $this->doLogQueryTimer($this->strQuery, gasStatic::doingTime($startTime));

            // populate member variables with query data/info
            $this->data = $cursor->toArray();           // gets all data, as array, in one line instead of looping
            $this->count = (empty($this->data)) ? 0 : count($this->data);
            $this->recordsReturned = $this->count;
            $this->recordsInQuery = $this->recordsReturned;
            $this->state = STATE_SUCCESS;
            $this->status = true;

            // if the query returned no (not found) data
            if (0 == $this->count) {
                $this->state = STATE_NOT_FOUND;
                $this->eventMessages[] = INFO_QUERY_RETURNED_NO_DATA;
                $this->eventMessages[] = $this->strQuery;
            } else {
                // check for auditing and, if enabled for read events, publish an audit event to admin broker
                if ($this->useAuditing == AUDIT_NONDESTRUCTIVE) {
                    $this->auditRecordList = $this->data;
                    if (!$this->registerAuditEvent(EVENT_NAME_AUDIT_FETCH)) {
                        $this->eventMessages[] = ERROR_AUDIT_GENERIC_FAIL;
                        consoleLog($this->res, CON_ERROR, ERROR_AUDIT_GENERIC_FAIL);
                    }
                }
                // check to see if the filterData bypass was set
                if (isset($this->metaPayload[META_DONUT_FILTER]) and $this->metaPayload[META_DONUT_FILTER] == 1 and $this->metaPayload[META_CLIENT] == CLIENT_SYSTEM) return;
                // filter the return data set and cache if caching is enabled
                if (!$this->returnFilteredData())
                    $this->eventMessages[] = ERROR_DATA_PROCESSING;
            }
        } catch (MongoDB\Driver\Exception\InvalidArgumentException  |
                 MongoDB\Driver\Exception\ConnectionException       |
                 MongoDB\Driver\Exception\AuthenticationException   |
                 MongoDB\Driver\Exception\RuntimeException          |
                 Throwable $t)                                      {
            $msg = sprintf(INFO_LOC, basename(__METHOD__),__LINE__) . ERROR_EXCEPTION;
            $this->eventMessages[] = $msg;
            $this->logger->warn($msg);
            $this->logger->warn($t->getMessage());
            $this->state = STATE_DB_ERROR;
        }
    }


    protected function _lockRecord()
    {

    }

    protected function _releaseLock()
    {

    }

    protected function _isLocked()
    {

    }

    public function _getQC(array $_data): bool
    {
        // TODO: Implement _getQC() method.
        $this->eventMessages[] = 'You need to write this method: ' . __METHOD__;
        $this->recordsInQuery = 2214;
        return true;
    }


    /**
     * processMetricsForGraphs() -- public function
     *
     * This function was added when the log broker was converted over to a parallel-processing log exchange.  The
     * purpose of this method is to take a metrics payload, validate the content, and copy specific fields from the
     * metrics payload into a container that can be used to create a graphs record.
     *
     * There is one input parameter to the method:
     *
     * $_data -- this is the $request[BROKER_DATA] array containing payload data from the metrics event
     *
     * The method returns a boolean indicating if the metrics event was successfully converted to graphs data and
     * then written to the graphs collection.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     * @return bool
     *
     * HISTORY:
     * ========
     * 10-07-19     mks     DB-136: code and testing completed
     *
     */
    public function processMetricsForGraph(array $_data): bool
    {
        $res = 'mPMG: ';
        $graphData = null;
        $data = $_data[0];
        if (!isset($data[STRING_EVENT . COLLECTION_MONGO_METRICS_EXT])) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            consoleLog($res, CON_ERROR, $hdr . ERROR_DATA_KEY_404 . STRING_EVENT . COLLECTION_MONGO_METRICS_EXT);
            return false;
        }
        // transfer the metrics payload over to the current object $data member
        $metFields = [
            LOG_LEVEL . COLLECTION_MONGO_METRICS_EXT => GRAPH_KEY,
            LOG_FILE . COLLECTION_MONGO_METRICS_EXT => GRAPH_FILE,
            LOG_METHOD . COLLECTION_MONGO_METRICS_EXT => GRAPH_METHOD,
            LOG_LINE . COLLECTION_MONGO_METRICS_EXT => GRAPH_LINE,
            LOG_MESSAGE . COLLECTION_MONGO_METRICS_EXT => GRAPH_COMMENT,
            DB_EVENT_GUID . COLLECTION_MONGO_METRICS_EXT => DB_EVENT_GUID,
            LOG_TIMER . COLLECTION_MONGO_METRICS_EXT => GRAPH_TIMER,
        ];
        $metaExceptions = [
            LOG_FILE . COLLECTION_MONGO_METRICS_EXT,
            LOG_METHOD . COLLECTION_MONGO_METRICS_EXT,
            LOG_LINE . COLLECTION_MONGO_METRICS_EXT
        ];
        $haveFile = false;
        $haveMethod = false;
        $haveLine = false;
        foreach ($metFields as $metaField => $graphField) {
            if (in_array($metaField, $metaExceptions)) {
                switch ($metaField) {
                    case LOG_FILE . COLLECTION_MONGO_METRICS_EXT :
                        $haveFile = true;
                    break;
                    case LOG_METHOD . COLLECTION_MONGO_METRICS_EXT :
                        $haveMethod = true;
                    break;
                    case LOG_LINE . COLLECTION_MONGO_METRICS_EXT :
                        $haveLine = true;
                    break;
                }
            } elseif (isset($data[$metaField])) {
                // good data - copy it into the local variable $graphData removing the metrics extension as we do so
                $graphData[$graphField] = $data[$metaField];
            } elseif (!isset($this->metaPayload[META_CLIENT]) or $this->metaPayload[META_CLIENT] != CLIENT_SYSTEM) {
                // if we're missing a key data field, we're not going to use this data to create a new record
                // unless the request is system request...
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $msg = $hdr . ERROR_DATA_KEY_404 . $metaField;
                consoleLog($res, CON_ERROR, $msg);
                return false;
            }
        }
        // check to see if we can build a location string
        if ($haveFile && $haveMethod && $haveLine) {
            $location[GRAPH_FILE] = $data[LOG_FILE . COLLECTION_MONGO_METRICS_EXT];
            $location[GRAPH_METHOD] = $data[LOG_METHOD . COLLECTION_MONGO_METRICS_EXT];
            $location[GRAPH_LINE] = $data[LOG_LINE . COLLECTION_MONGO_METRICS_EXT];
            $graphData[GRAPH_LOCATION] = $location;
        } else {
            // error messages about not having all the location bits
            if (!$haveFile)
                consoleLog($res, CON_ERROR, ERROR_DATA_KEY_404 . LOG_FILE . COLLECTION_MONGO_METRICS_EXT);
            if (!$haveMethod)
                consoleLog($res, CON_ERROR, ERROR_DATA_KEY_404 . LOG_METHOD . COLLECTION_MONGO_METRICS_EXT);
            if (!$haveLine)
                consoleLog($res, CON_ERROR, ERROR_DATA_KEY_404 . LOG_LINE . COLLECTION_MONGO_METRICS_EXT);
            return false;
        }
        // save the graph record data
        try {
            $this->_createRecord([$graphData]);
        } catch (Throwable | TypeError $t) {
            foreach ($this->eventMessages as $errorMessage)
                consoleLog($res, CON_ERROR, $errorMessage);
            $hdr =  basename(__METHOD__) . AT . __LINE__ . COLON;
            consoleLog($res, CON_ERROR, $hdr . ERROR_EXCEPTION);
            consoleLog($res, CON_ERROR, $hdr . $t->getMessage());
            return false;
        }
        return true;
    }


    /**
     * validateSubCollectionData() -- private method
     *
     * this method is called by the two sub-collection processing methods to avoid code duplication.
     *
     * There are three inputs to the method:
     * $_pk     -- the primary key value which must be a GUID
     * $_field  -- the sub-collection field name
     * $_data   -- the sub-collection record(s) that will be added
     *
     * Checks that we're making to validate the input:
     *
     * 1. that the pk DB_TOKEN field is a member of the current class
     * 2. that the pk is a valid GUID
     * 3. build a fetch query using the pk to validate that the referenced record exists
     * 4. that the _field parameter is both a member of the class and defined as a subCollection
     * 5. validate the record(s) array by passing them through the cache mapper
     *
     * if any of the tests fail processing, we stop processing an immediately return a Boolean(false) value,
     * otherwise, we'll inject a guid into the SUBC record, and return a true value to indicate that the input
     * parameters passed validation.
     *
     * @param string $_pk
     * @param string $_field
     * @param array $_data
     * @return bool
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * HISTORY:
     * ========
     * 08-28-16     mks     CORE-494: initial coding
     * 03-14-18     mks     CORE-833: (re)injecting guids into sub-collection record data
     * 11-26-18     mks     DB-55: fixed bug - checking array values instead of array content
     * 02-04-19     mks     DB-107: fixed PHP warning assuming comparison array is an array
     * 02-15-19     mks     DB-116: deprecated cache-mapping; now handled at broker-services level
     *
     */
    private function validateSubCollectionData(string $_pk, string &$_field, array &$_data): bool
    {
        // step 1: validate the sub-array members
        if (!array_key_exists(0, $_data)) {
            $msg = ERROR_DATA_ARRAY_NOT_ARRAY . STRING_DATA;
            $this->eventMessages[] = $msg;
            $this->state = STATE_DATA_ERROR;
            return false;
        }
        try {
            if (is_null($_data)) {
                $msg = ERROR_CACHE_MAP_FAIL . STRING_DATA;
                $this->eventMessages[] = $msg;
                $this->state = STATE_DATA_ERROR;
                return false;
            }

            // Step 2: generate a GUID for each sub-collection record if there is not one already set
            foreach ($_data as &$record) {
                // generate a GUID for the subC record if one doesn't already exist
                if (is_array($record) and !array_key_exists((DB_TOKEN . $this->ext), $record)) {
                    $record[(DB_TOKEN . $this->ext)] = guid();
                }
            }

            // step 3: validate the PK (param 1) value
            if (!validateGUID($_pk)) {
                $msg = ERROR_INVALID_GUID . $_pk;
                $this->eventMessages[] = $msg;
                $this->logger->data($msg);
                return false;
            }

            // step 4: fetch the record reference by $_pk and populate the $data property
            $fetchQuery = [STRING_QUERY_DATA => [DB_TOKEN => [OPERAND_NULL => [OPERATOR_EQ => [$_pk]]]]];
            $oc = $this->useCache;
            $this->useCache = false;        // disable caching so we don't waste time caching the data pre-update
            $this->_fetchRecords($fetchQuery);
        } catch (Throwable $t) {
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->error($msg);
            else
                consoleLog($this->res, CON_ERROR, $msg);
            $this->state = STATE_FRAMEWORK_WARNING;
            return false;
        }
        $this->useCache = $oc;
        // if the pk search returned no data, return a 404 message
        if ($this->state == STATE_NOT_FOUND or !$this->status) {
            $msg = ERROR_UT_QUERY_RETURNED_ZERO;
            $this->eventMessages[] = $msg;
            return false;
        }

        // step 5: validate the subC value (parameter 2) as a member AND a subC field -- compensate for the user
        //         accidentally submitting a schema key, with or without the class extension
        if (is_array($this->cacheMap) and in_array($_field, $this->cacheMap)) {
            $_field = array_search($_field, $this->cacheMap);
        } /** @noinspection PhpStatementHasEmptyBodyInspection */ elseif (in_array($_field, $this->fieldList)) {
            // nothing to do
        } elseif (in_array(($_field . $this->ext), $this->fieldList)) {
            $_field = $_field . $this->ext;
        } else {
            $msg = sprintf(ERROR_DATA_INVALID_CLASS_KEY, $_field, $this->class);
            $this->eventMessages[] = $msg;
            $this->state = STATE_DATA_ERROR;
            return false;
        }

        if (!array_key_exists($_field, $this->subCollections)) {
            $msg = ERROR_SUB_COLLECTION_NOT_MEMBER . $_field;
            $this->eventMessages[] = $msg;
            $this->state = STATE_DATA_ERROR;
            return false;
        }
        return true;
    }


    /**
     * doBulkWrite() -- private method
     *
     * this method executes a mongoDB bulkWrite command.  The input parameter to the method is a pre-declared and
     * instantiated BulkWrite class object.
     *
     * Bulk-writes can be constructed with one, or more, destructive operations -- the driver will attempt to send
     * operations of the same type in as few requests as possible in order to optimize round trips.
     *
     * This functionality was broken out into it's own method because, even though the basic command can exist in
     * several locations throughout this class, writing the exception handling becomes tiring.
     *
     * This method updates several class properties with the query results:
     *
     * $this->status:  set to Boolean true on success, o/w: false
     * $this->state :  set to STATE_SUCCESS on success, o/w: defaults to DB_ERROR
     * $this->queryResults:  results of the bulk write request in terms of documents affected by operation
     *
     * $this->eventMessages will have new errors, if any, appended.
     * $this->strQuery should be populated PRIOR to invoking this method.
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param MongoDB\Driver\BulkWrite $_bwo
     *
     * HISTORY:
     * ========
     * 07-19-17     mks     CORE-464: pulled from _update() method into it's own method to be called for all class
     *                                bulkWrite requests
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     *
     */
    private function doBulkWrite(MongoDB\Driver\BulkWrite $_bwo):void
    {
        $this->status = false;
        $this->state = STATE_DB_ERROR;
        $startTime = floatval(0);

        try {
            if (!is_object($this->manager) or empty($this->manager) or is_null($this->manager)) {
                $this->state = STATE_FRAMEWORK_FAIL;
                $this->eventMessages[] = ERROR_RESOURCE_404 . $this->dbService;
                $this->logger->fatal(sprintf(basename(__METHOD__), __LINE__) . ERROR_RESOURCE_404 . $this->dbService);
                return;
            }
            if ($this->useTimers) $startTime = gasStatic::doingTime();
            /** @var MongoDB\Driver\WriteResult $result */
            $result = $this->manager->executeBulkWrite($this->nameSpace, $_bwo, $this->writeConcern);
            $msg = sprintf(
                MONGO_BULK_WRITE_RESULTS,
                $result->getMatchedCount(), $result->getModifiedCount(),
                $result->getUpsertedCount(), $result->getInsertedCount(),
                $result->getDeletedCount()
            );
            if ($this->useTimers) $this->doLogQueryTimer(($this->strQuery . COLON . $msg), gasStatic::doingTime($startTime));
            $this->bwResult = $result;
            $this->state = STATE_SUCCESS;
            $this->status = true;
            $this->rowsAffected = $result->getMatchedCount();
            $this->queryResults[] = $msg;
            if ($this->debug) {
                $this->logger->debug($this->strQuery);
                $this->logger->debug($msg);
            }
        } catch (MongoDB\Driver\Exception\BulkWriteException | MongoDB\Driver\Exception\RuntimeException |
                 MongoDB\Driver\Exception\ConnectionTimeoutException | Throwable $e) {
            $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
            $this->state = STATE_DB_ERROR;
//            $res = $e->getWriteResult();
//            // write concern could not be fulfilled
//            if (null != $res->getWriteConcernError()) {
//                $msg = sprintf("%s  (%d): %s", $e->getWriteResult()->getWriteConcernError()->getMessage(),
//                    $e->getWriteResult()->getWriteConcernError()->getCode(),
//                    $e->getWriteResult()->getWriteConcernError()->getInfo(), true);
//            } else {
//                $msg = $e->getMessage();
//            }
            $this->eventMessages[] = $hdr . ERROR_MONGO_EXCEPTION_BW_EXEC;
            $this->eventMessages[] = $hdr . STRING_QUERY . COLON . $this->strQuery;
            $this->eventMessages[] = $hdr . $e->getMessage(); //$msg;
            $this->logger->warn($hdr . ERROR_MONGO_EXCEPTION_BW_EXEC);
//            $this->logger->warn($hdr . STRING_QUERY . COLON . $this->strQuery);
            $this->logger->warn($hdr . $e->getMessage());  // used to be: $msg
            // check if any write operations failed to complete
//            foreach ($res->getWriteErrors() as $writeError) {
//                $msg = sprintf("Op #(%d): %s (%d)", $writeError->getIndex(), $writeError->getMessage(), $writeError->getCode());
//                $this->eventMessages[] = $msg;
//                $this->logger->warn($hdr . $msg);
//            }
        }
    }


    /**
     * queryBuilder() -- private method
     *
     * Doc:  https://givingassistant.atlassian.net/wiki/display/INF/Namaste+Data+Payloads
     *
     * this method requires a single input: the array containing the query elements as submitted by the client.
     *
     * This method has it's own wiki page - please refer to the wiki page for implementation/usage details.
     *
     * Basically, the query builder both processed and validates the query portion of a client query request.
     * (Other methods handle validation of the projection and the extras.)
     *
     * For each level of the query tree, we start a loop for every element in that level.  Each level is validated
     * appropriately.  For example, at the attribute level, we validate that the attribute is a declared index
     * for the current class and, at the value level, we validate that the value type matches the type
     * declared for that attribute.
     *
     * When we're processing a query tree with more than one branch, we store each branch in a collective array until
     * we encounter an operand at the attribute level.  The presence of the operand declares that the previous
     * branches will be joined by this operand.  This is stored in $complexQuery.
     *
     * Any errors, or rules violations, will cause an error message to be generated and copied to the eventMessages
     * property and output to the log file, and execution will immediately return back to the calling client and,
     * instead of an array return value, a null will be sent back to the client.
     *
     * The over success/fail of the method can be determined by:
     *
     *     -- state property is set to 'success'
     *     -- status property is set to true
     *     -- a non-null array is returned
     *
     * If all of the above conditions are not met, then query building failed.
     *
     * NOTES:
     * ------
     * Preparing for complex queries at a later time, here's an example of what we'll try to achieve with version 2
     * of the query-builder algorithm (embedding operands at level 3):
     *
     * "$or": [{
     *     "$and": [{
     *         "class": "a"
     *     }, {
     *         "rate": {
     *             "$gt": 20000
     *         }
     *     }]
     * }, {
     *     "$and": [{
     *         "class": "b"
     *     }, {
     *         "multiply": {
     *             "$gt": 20000
     *         }
     *     }]
     * }]
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_query
     * @return array|null
     *
     * HISTORY:
     * ========
     * 07-12-17     mks     CORE-464: initial coding
     * 09-11-17     mks     CORE-558: removed N%1 test as is no longer a valid test
     * 03-27-19     mks     DB-116: support for NIN and IN operators for complex queries
     *
     */
    private function queryBuilder(array $_query): ?array
    {
        $this->state = STATE_VALIDATION_ERROR;
        $this->status = false;
        $query = '';
        $queryThreads = null;
        $joinOperand = null;
        $complexQuery = null;

        $operatorExceptions = [
            OPERATOR_IN,
            OPERATOR_NIN
        ];
        $validOperands = [
            OPERAND_AND     => MONGO_AND,
            OPERAND_OR      => MONGO_OR,
            OPERAND_NOT     => MONGO_NOT,
            OPERAND_NOR     => MONGO_NOR,
            OPERAND_NULL    => OPERAND_NULL
        ];
        $opCodes = [
            OPERATOR_EQ     => MONGO_EQ,
            OPERATOR_DNE    => MONGO_DNE,
            OPERATOR_NIN    => MONGO_NIN,
            OPERATOR_IN     => MONGO_IN,
            OPERATOR_GT     => MONGO_GT,
            OPERATOR_GTE    => MONGO_GTE,
            OPERATOR_LTE    => MONGO_LTE,
            OPERATOR_LT     => MONGO_LT,
            OPERATOR_REGEX  => MONGO_REGEX,
            OPERAND_AND     => MONGO_AND,
            OPERAND_NOT     => MONGO_NOT,
            OPERAND_OR      => MONGO_OR,
            OPERAND_NOR     => MONGO_NOR
        ];

        // validate the query array
        if (!is_array($_query)) {
            $msg = ERROR_DATA_MISSING_ARRAY . STRING_QUERY;
            $this->eventMessages[] = $msg;
            $this->logger->data($msg);
            return null;
        }
        // start parsing the array beginning with the attribute elements
        foreach ($_query as $attribute => $operandLevel) {
            $query = '';
            $tmp = null;
            // validate the attribute name as being a member of the current class
            $fieldName = '';
            // test to see if we're working with a join operand (3 or more branches in the query tree)
            if (array_key_exists($attribute, $validOperands) and is_null($operandLevel)) {
                $joinOperand = $attribute;
                // we've detected a join operand: all previous queries have to be arranged under the root-level operand
                if (is_array($complexQuery) and array_key_exists($joinOperand, $complexQuery)) {
                    // duplicate root-level operand - generate error and return
                    $msg = sprintf(ERROR_QB_ROOT_OPERANDS, $joinOperand);
                    $this->eventMessages[] = $msg;
                    $this->logger->data($msg);
                    return null;
                }
                $complexQuery= [ $validOperands[$joinOperand] => $queryThreads];
                $queryThreads = null;
                break;
            }
            // step 1 -- is in the field list?  Check to see if this is non-mapped value:
            if (in_array(($attribute . $this->ext), $this->fieldList)) {
                $fieldName = $attribute . $this->ext;
            } elseif (in_array($attribute, $this->fieldList)) {
                $fieldName = $attribute;
            } elseif (in_array($attribute, $this->cacheMap)) {
                $fieldName = array_search($attribute, $this->cacheMap);
            }
            if ($fieldName == '') {
                $msg = sprintf(ERROR_QB_ATTRIBUTE_404, $attribute, $this->class);
                $this->eventMessages[] = $msg;
                $this->logger->data($msg);
                $this->state = STATE_DATA_ERROR;
                return null;
            }
            // validate that the fieldName is an indexed discriminant -- if not, reject the query
            if (!in_array($fieldName, $this->indexes)) {
                $msg = sprintf(ERROR_QB_NOT_INDEXED_KEY, $attribute, $this->class);
                $this->eventMessages[] = $msg;
                $this->logger->data($msg);
                $this->state = STATE_INDEX_ERROR;
                return null;
            }
            // we've validated the attribute name - append the extension and move into the operand processing
            foreach ($operandLevel as $operand => $operatorLevel) {
                if (!array_key_exists($operand, $validOperands)) {
                    $msg = ERROR_QB_INVALID_OPERAND . $operand;
                    $this->eventMessages[] = $msg;
                    $this->logger->error($msg);
                    $this->state = STATE_DATA_ERROR;
                    return null;
                } elseif (count($operatorLevel) != 1 and $operand == OPERAND_NULL) {
                    $msg = sprintf(ERROR_QB_VALUE_COUNT, $operand, 1);
                    $this->eventMessages[] = $msg;
                    $this->logger->error($msg);
                    $this->state = STATE_DATA_ERROR;
                    return null;
                } else {
                    $op = $validOperands[$operand];
                }
                foreach ($operatorLevel as $operator => $values) {
                    if ($operand == OPERAND_NULL and is_array($values)) {
                        // exception case: we're peeking ahead to see if $operator = OPERATOR_IN | OPERATOR_NIN
                        if (in_array($operator, $operatorExceptions)) {
                            $query = [ $fieldName => [ $opCodes[$operator] => $values[0] ]];
                        } else {
                            // validate the data type of the request against the registered field type
                            if (gettype($values[0]) != $this->fieldTypes[$fieldName]) {
                                $msg = sprintf(ERROR_QB_TYPE_MISMATCH, $this->fieldTypes[$fieldName], (string)$values[0], gettype($values[0]));
                                $this->eventMessages[] = $msg;
                                $this->logger->data($msg);
                                $this->state = STATE_DATA_ERROR;
                                return null;
                            }
                            $query = [$fieldName => [$opCodes[$operator] => $values[0]]];
                        }
                    } else {
                        // { $or : [ { level_log : {$eq : 'warning'}}, {level_log: {$eq: 'fatal'}}]}
                        foreach ($values as $value) {
                            $tmp[] = [$fieldName => [$opCodes[$operator] => $value]];
                        }
                    }
                }
                if (empty($query) and !empty($tmp) and is_array($tmp))
                    $query = [ $op => $tmp];
            }
            $queryThreads[] = $query;
        }
        if (!is_null($queryThreads) and !is_null($complexQuery)) {
            $this->eventMessages[] = ERROR_QB_ROOT_OPERAND_404;
            $this->logger->data(ERROR_QB_ROOT_OPERAND_404);
            return null;
        }
        $this->state = STATE_SUCCESS;
        $this->status = true;
        return($complexQuery ?? $query);
    }


    /**
     * sortBuilder() -- private method
     *
     * This method requires a single input parameter: the sort array as received from the broker payload under
     * the keys:  BROKER_DATA -> STRING_SORT_DATA.
     *
     * The expected array is an associative array containing a list of field names referencing a sort-direction,
     * either STRING_SORT_ASC or STRING_SORT_DESC.
     *
     * The method will spin through the sort data and will check to see if the current class has enabled cache-mapping.
     * If so, we'll search for the field in the cache-map (values) and, if found, will extract the key.  If cache
     * mapping is not being used, we'll search the current class field-list instead.
     *
     * If found, validate the sort-direction converting the string value to mongo-friendly integers.  If the sort key
     * does match the range(allowed-values), generate an error message and exit.
     *
     * Method returns a sort array that's directly compatible with mongo, or a null.  Since the calling client checks
     * to see if the sort array is non-null prior to submission, errors in data are going to be user-errors 99%
     * of the time.
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_sortData
     * @return array|null
     *
     * HISTORY:
     * ========
     * 07-17-17     mks     CORE-464: initial coding
     *
     */
    private function sortBuilder(array $_sortData): ?array
    {
        $sortArray = [];
        $validDirs = [ STRING_SORT_ASC, STRING_SORT_DESC];
        if (!is_array($_sortData) or empty($_sortData)) {
            $this->eventMessages[] = ERROR_MDB_SORT_ARRAY_NOT;
            $this->logger->data(ERROR_MDB_SORT_ARRAY_NOT);
            return null;
        }
        foreach ($_sortData as $field => $sortDir) {
            $mappedField = '';
            if ($this->useCache and is_array($this->cacheMap)) {
                if (in_array($field, $this->cacheMap)) {
                    $mappedField = array_search($field, $this->cacheMap);
                } else {
                    $msg = sprintf(ERROR_MDB_FIELD_NOT_CACHED, $field, $this->class);
                    $this->eventMessages[] = $msg;
                    $this->logger->info($msg);
                }
            } else {
                if (in_array(($field . $this->ext), $this->fieldList)) {
                    $mappedField = $field . $this->ext;
                }
            }
            if (!empty($mappedField) and in_array($sortDir, $validDirs)) {
                $sortArray[$mappedField] = ($sortDir == STRING_SORT_ASC) ? 1 : -1;
            } elseif (!empty($mappedField)) {
                $msg = ERROR_MDB_SORT_DIR_404 . $sortDir;
                $this->eventMessages[] = $msg;
                $this->logger->data($msg);
                return null;
            }
        }
        return($sortArray);
    }


    /**
     * updateBuilder() -- private method
     *
     * The purpose of this function is to spin through the UPDATE_DATA payload array and ensure that each key in the
     * array has the class extension appended.
     *
     * There is a single input parameter to the method:
     *
     * $_data:  this is a call-by-reference associative array containing the STRING_UPDATE_DATA payload.
     *
     * If the array is processed successfully, then we'll replace the array keys with the new values and append
     * the class extension to the key.
     *
     * There's also checks made against the key vs. the protectedFields list to ensure that we're not attempting to
     * update a field, like createdDate, that's on the protectedField list.
     *
     * If there's a protected fields violation, we'll return a boolean(false) and set the STATE member variable to
     * STATE_VALIDATION_ERROR. If the key is not found in the cacheMap or the fieldList, then we'll return a
     * boolean(false) and set the state member to STATE_NOT_FOUND.
     *
     * If a false value is returned, the input parameter will not be updated.
     *
     * Otherwise a boolean(true) is returned and the input parameter is replaced with the new array.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     * @return void
     *
     *
     * HISTORY:
     * ========
     * 03-14-19     mks     DB-116: original coding
     *
     */
    private function updateBuilder(array &$_data): void
    {
        $newArray = null;
        // audit and unit-test client processes are allowed to update protected fields
        $isAudit = (isset($this->metaPayload[META_CLIENT])) ? ($this->metaPayload[META_CLIENT] == CLIENT_AUDIT) : false;
//        $isUT = (isset($this->metaPayload[META_CLIENT])) ? ($this->metaPayload[META_CLIENT] == CLIENT_UNIT) : false;
        foreach ($_data as $key => $value) {
            // first - test for protected field violations and exempt CLIENT_AUDIT from check
            if (!$isAudit and in_array($key, $this->protectedFields)) {
                $msg = sprintf(ERROR_QB_PF_VIOL, $key) . $this->metaPayload[META_CLIENT];
                $this->eventMessages[] = $msg;
                unset($newArray[$key]);
            } elseif (in_array($key, $this->fieldList)) {
                $newArray[$key] = $value;
            } elseif (in_array(($key . $this->ext), $this->fieldList)) {
                $newArray[($key . $this->ext)] = $value;
            } elseif (is_array($this->cacheMap) and in_array($key, $this->cacheMap)) {
                // this code is here for when we're testing and bypassing the brokers
                // key is in cacheMap
                $newKey = array_search($key, $this->cacheMap);
                // ensure cacheMapped-translated key is not in protected fields
                if (!$isAudit and in_array($newKey, $this->protectedFields)) {
                    $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                    $msg = sprintf(ERROR_QB_PF_VIOL, $key);
                    $this->eventMessages[] = $msg;
                    $this->logger->data($hdr . $msg);
                    $this->state = STATE_VALIDATION_ERROR;
                }
                $newArray[$newKey] = $value;
            } else {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $msg = ERROR_KEY_404 . $key;
                $this->eventMessages[] = $msg;
                $this->logger->data($hdr . $msg);
                $this->state = STATE_NOT_FOUND;
            }
        }
        $_data = (empty($newArray)) ? null : $newArray;
    }


    /**
     * projectionBuilder() -- private method
     *
     * This method takes a single, required, input parameter -- which should be the indexed array of data received
     * by the broker (BROKER_DATA -> STRING_RETURN_DATA) which should be a list of fields that the user wants
     * returned from the query requested.
     *
     * We'll check to make sure we're dealing with a non-empty array.  If caching is enabled for the class, build the
     * projection from the cacheMap keys.  If caching isn't enabled, then scan the class fieldList property for
     * the field names.
     *
     * Create an associative array consisting of field name => 1 to indicate that this field should be returned.
     * Because the input data is derived from a formula generic for all database schemas, we're not looking for
     * mongo feature of either specifying the field be either omitted or included.  Any field listed is assumed
     * to be included.
     *
     * Unless the input array is empty, we'll never return a null for the projection as we'll always remove the
     * mongo pkey (_id) field from the return data set.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_data
     * @return array
     *
     * HISTORY:
     * ========
     * 07-17-17     mks     CORE-464: original coding
     * 04-01-20     mks     PD-21: fixed processing error by re-ordering processing logic for projection fields that
     *                             are either still represented by their cacheMap value, have or do not have the class
     *                             extension appended to the $field
     *
     */
    private function projectionBuilder(array $_data): array
    {
        $projection = [ MONGO_ID => 0 ];                               // by default, we always hide the mongo object id

        if (!is_array($_data) or empty($_data)) {                 // by default, we have to return at-least the db-token
            return $projection;                                   // don't "fix" this by adding the db_token value...
        }

        foreach ($_data as $field) {
            if ((substr($field, -NUMBER_LEN_CLASS_EXT, NUMBER_LEN_CLASS_EXT) == $this->ext)) {
                if (in_array($field, $this->fieldList)) {
                    $projection[$field] = 1;
                }
            } elseif (in_array(($field . $this->ext), $this->fieldList)) {
                $projection[($field . $this->ext)] = 1;
            } elseif ($this->useCache and is_array($this->cacheMap)) {
                if (in_array($field, $this->cacheMap)) {
                    $projection[array_search($field, $this->cacheMap)] = 1;
                }
            } else {
                $msg = ERROR_DATA_FIELD_NOT_MEMBER . $field;
                $this->eventMessages[] = $msg;
                $this->logger->error($msg);
            }
        }

        // ensure we've added the token to any projection otherwise, if enabled, caching will fail b/c no cache key
        if (!array_key_exists((DB_TOKEN . $this->ext), $projection) and in_array((DB_TOKEN . $this->ext), $this->fieldList)) {
            $projection[(DB_TOKEN . $this->ext)] = 1;
        }
        return($projection);
    }


    /**
     * gwValidateQuery() -- public gateway method
     *
     * this method allows testing of the query builder via unitTesting or script stubs.  The input parameter is an
     * array of query data that will be passed directly to the query builder method and the return from the method
     * is passed directly back to the calling client.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param $_query
     * @return null|array
     *
     * HISTORY:
     * ========
     * 07-11-17     mks     original coding
     *
     */
    public function gwValidateQuery($_query): ?array
    {
        return($this->queryBuilder($_query));
    }/** @noinspection PhpUnused */


    /**
     * gwBulkWrite() -- public gateway method
     *
     * this is a gateway function to prepareBulkWriteInsertData() so that the method can be invoked from a broker
     * for a system event.
     *
     * The method takes no input parameters and returns void -- success and failure are tracked using class members.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     *
     * HISTORY:
     * ========
     * 02-06-18     mks     _INF-139: original coding
     *
     */
    public function gwBulkWrite(): void
    {
        $this->prepareBulkWriteInsertData();
    }


    /**
     * fetchQueryTokenList() -- private method
     *
     * This method requires one input parameter:
     *
     * $_query -- this is the query, in array format, that was returned from queryBuilder() in the calling client.  In
     * other words, the query is pre-built and validated.
     *
     * The method has two exception handled -- one to handle the typeError exception for calling the queryBuilder
     * method and the second to handle the mongoDB API calls.
     *
     * In the event an exception is raised, diagnostics are stored in the eventMessages container and a boolean
     * false value is returned to the calling client.
     *
     * Otherwise, the internal member field: $auditRecordList, is populated with copies of the data pre-modification
     * and a boolean true is returned to the calling client.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param array $_query
     * @return bool
     *
     * HISTORY:
     * ========
     * 10-31-18     mks     DB-68:  original coding
     * 02-19-19     mks     DB-116: converted to (array) cursor-fetch over expensive/slow looping
     * 06-20-19     mks     DB-122: enforcing LIMIT in the return data set
     * 10-11-19     mks     DB-136: fixed bug: wasn't copying records into tokenRecordList member on cases where
     *                              the return data set count was less than $limit
     * 01-21-20     mks     DB-150: fixed a bug where $auditRecordList was not being set on the else condition (if the
     *                              sizeOf(list) was greater than the allowed number of records returned.
     *
     */
    private function fetchRecordsBeforeChange(array $_query): bool
    {
        // get the number of records to return and store this integer value in $limit
        $limit = intval((isset($this->metaPayload[META_LIMIT])) ? $this->metaPayload[META_LIMIT] : gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT]);
        // if the meta limit exceeds the framework limit, reduce the value to the f/w limit
        if ($limit > gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT])
            $limit = gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_RECORD_LIMIT];
        // we'll store the returned records in this member container:
        $this->auditRecordList = [];
        $recordList = null;
        $startTime = floatval(0);
        // build the query objects and submit
        try {
            $options = [ STRING_LIMIT => $limit, STRING_PROJECTION => [ MONGO_ID => 0 ]];
            $queryObject = new MongoDB\Driver\Query($_query, $options);
            $queryString = json_encode($_query);
            // start the query timer
            if ($this->useTimers) $startTime = gasStatic::doingTime();
            // get the cursor object from the query execution and populate the $data property
            /** @var Cursor $cursor */
            $cursor = $this->manager->executeQuery($this->nameSpace, $queryObject, $this->readPreference);
            // stop the query timer
            if ($this->useTimers)
                $this->doLogQueryTimer($queryString, gasStatic::doingTime($startTime));
            $cursor->setTypeMap(['root' => 'array', 'document' => 'array', 'array' => 'array']);
            $recordList = $cursor->toArray();      // gets all data, as array, in one line instead of looping
            if (empty($recordList)) {
                $this->state = STATE_NOT_FOUND;
                return false;
            }
            // if the captured list exceeds the record limit, reduce the return payload to the $limit size
            if (is_array($recordList) and count($recordList) > $limit) {
                $this->auditRecordList = array_slice($recordList, 0, $limit, true);
                if (empty($this->auditRecordList)) {
                    // we could not, for w/e reason, extract the limit-based slice so restore to original payload
                    $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                    consoleLog($this->res, CON_ERROR, $hdr . ERROR_DATA_ARRAY_SLICE);
                    $this->eventMessages[] = ERROR_DATA_ARRAY_SLICE;
                    $this->auditRecordList = $recordList;
                    return false;
                } else {
                    $this->recordTokenList = array_column($this->auditRecordList, ($this->pKey . $this->ext));
                    $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                    $this->logger->info($hdr . sprintf(ERROR_DATA_ARRAY_SLICE_INFO, count($recordList), $limit));
                }
            } elseif (is_array($recordList)) {
                $this->recordTokenList = array_column($recordList, ($this->pKey . $this->ext));
                $this->auditRecordList = $recordList;
            }
        }  catch (MongoDB\Driver\Exception\InvalidArgumentException |
                  MongoDB\Driver\Exception\ConnectionException |
                  MongoDB\Driver\Exception\AuthenticationException |
                  MongoDB\Driver\Exception\RuntimeException |
                  TypeError | Throwable $e) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_MONGO_EXCEPTION . $e->getMessage();
            if (isset($this->logger) and $this->logger->available)
                $this->logger->warn($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->eventMessages[] = $msg;
            return false;
        }
        return true;
    }


    /**
     * prepareBulkWriteInsertData() -- private method
     *
     * prepareBulkWriteInsertData (code) used to be in the _create() method but was broken out into a stand-alone
     * private method in order to accommodate system-level requests to add data to a collection avoiding redundant
     * class/data validation.  (Which should use the gwBulkWrite() public method for access.)
     *
     * For this event to successfully complete, the following pre-processing is assumed:
     *
     * 1. that all the data is stored in the $data member as an indexed array of associative arrays
     * 2. that all the associative arrays have keys that exactly match the current collection's column names
     * 3. that all of the associative arrays have values of the correct type
     *
     * This method instantiates the BulkWrite mongoDB object and then loops through the records stored in $data and
     * performs an insert operation call to the BW object.  Note that we're selecting an "ordered" write meaning that
     * the records are added in the order that there appear in the $data member.  If the bulk-write fails, we can
     * look at the number of records written vs. the total records in the count and derive the last record written.
     *
     * The actual insertion is handled in doBulkWrite(), a class method.
     *
     * The method requires no input parameters and returns void.  Success or fail is indicated by the class state
     * and status members.
     *
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * HISTORY:
     * ========
     * 02-06-18     mks     _INF-139: original coding
     * 02-07-18     mks     _INF-139: PHP 7.2 exception handling, console logging of exception
     * 10-19-18     mks     DB-67: recovery queries for audit added
     * 10-29-18     mks     DB-74: undo-queries for journaling added
     *
     */
    private function prepareBulkWriteInsertData(): void
    {
        //--------------------------------------------------------------------------------------------------------------
        // set-up the write-concern as follows:
        //    -- stand-alone:  default (1)
        //    -- repl-set:     {string} -- the replSet tag name: writes propagate to at least 1 member of the replSet
        //    -- shard:        mongos instances pass the write concern to the shards
        //
        // general rules/guidelines for mongo:
        // -----------------------------------
        // -- journaling enabled on all nodes for production  (w = MAJORITY)
        // -- journaling not enabled on admin for staging
        // -- journaling not enabled on development
        // -- stand-alone instances use default write-concern
        // -- replSets use the replSet tag for write-concern value
        //
        // with journaling set to true, mongo returns only after the requested number of members, including the
        // primary, have written to the journal.
        //
        // wtimeout values: if not specified, or set to 0, write operation could block indefinitely if the level
        //                  of the write concern is not achievable.
        //
        //                  wtimeout set in the config can be over-written with new values passed to the API.
        //                  todo: consider making the wtimeout value a scaling value of the data being written
        //
        // FINAL NOTE:  The write concern class is build in the constructor...
        //--------------------------------------------------------------------------------------------------------------

        // init the bulkWrite class object
        try {
            $fatso = new MongoDB\Driver\BulkWrite([MONGO_STRING_ORDERED => true]);   // ordered
            // add an insert operation, using each $data record, to the bulkWrite class object
            $update = [ DB_STATUS => STATUS_DELETED ];
            $tmpMeta = $this->metaPayload;
            if (isset($tmpMeta[META_SESSION_GUID])) unset($tmpMeta[META_SESSION_GUID]);
            // loop through each data record, add to the insertion, and process for audit/journaling
            foreach ($this->data as $record) {
                $fatso->insert($record);
                // build the audit query
                if ($this->useAuditing) {
                    $query = [DB_TOKEN => [OPERAND_NULL => [OPERATOR_EQ => [$record[DB_TOKEN . $this->ext]]]]];
                    // check and, if journaling is enabled, build the recovery queries
                    if ($this->useJournaling) {
                        $this->auditRecordList[] = $record[DB_TOKEN . $this->ext];
                        $this->tokenList[] = $record[DB_TOKEN . $this->ext];
                        if ($this->useDeletes) {                                                         // HARD DELETES
                            $this->auditUndoQueries[] = [
                                BROKER_REQUEST => BROKER_REQUEST_DELETE,
                                BROKER_DATA => [ STRING_QUERY_DATA => $query ],
                                BROKER_META_DATA => $tmpMeta
                            ];
                        } else {                                                                         // SOFT DELETES
                            $this->auditUndoQueries[] = [
                                BROKER_REQUEST => BROKER_REQUEST_UPDATE,
                                BROKER_DATA => [ STRING_QUERY_DATA => $query, STRING_UPDATE_DATA => $update ],
                                BROKER_META_DATA => $tmpMeta
                            ];
                        }
                    }
                    // build the audit query used to create the record(s)
                    $this->auditCreateQueries[] = [
                        BROKER_REQUEST => BROKER_REQUEST_CREATE,
                        BROKER_DATA => [$record],
                        BROKER_META_DATA => $tmpMeta
                    ];
                }
            }
        } catch (MongoDB\Driver\Exception\InvalidArgumentException |
                 MongoDB\Driver\Exception\ConnectionTimeoutException |
                 Throwable | TypeError $e) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $this->eventMessages[] = ERROR_EXCEPTION;
            @handleExceptionMessaging($hdr, $e->getMessage(), $foo, true);
            return;
        }
        // execute the bulkWrite
        $this->strQuery = sprintf(MONGO_QUERY_BULK_CREATE, $this->count);
        $this->doBulkWrite($fatso);
    }


    /**
     * loadTemplates() -- private method
     *
     * this method is invoked by the constructor and serves to load the class template file, assimilating it into
     * the current instantiation.
     *
     * the method will load the class template and set the class member variables controlled/referenced by the
     * template.
     *
     * NOTE:
     * -----
     * This is NOT the same processing-method as the gacFactory::fetchSchema method!
     *
     * successful loading of the template is determined by the return (boolean) value -- on error, a log message
     * will be generated so it's up to the developer to check logs on fail-returns to see why their template
     * file was not correctly assimilated.
     *
     * The only input parameter is the name of the template (as a class - so it can be instantiated).
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * @param string $_template
     * @return bool
     *
     * HISTORY:
     * ========
     * 07-10-17     mks     original coding
     * 08-03-17     mks     CORE-467: support 3.2 indexes and index properties
     *                                added additional error checking for validation of index column names
     * 10-02-17     mks     CORE-572: bug-fix: exposed fields missing extension
     * 02-01-18     mks     _INF-139: importing migration data
     * 10-17-18     mks     DB-59:    journaling and auditing
     * 10-31-19     mks     DB-136:   fixed the validation and processing for partialIndexes according to the "new"
     *                                template format.
     *
     */
    private function loadTemplates(string $_template): bool
    {
        // DO NOT PUT A PGS-ERROR_SET CALL IN THIS METHOD!

        try {
            $this->template = new $_template();
            if (!is_object($this->template)) {
                $this->logger->warn(ERROR_FILE_404 . $_template);
                $this->setState(ERROR_FILE_404 . $_template);
                return (false);
            }

            $tmpMeta = new gacMeta();
            if (!is_object($tmpMeta)) {
                $this->logger->warn(ERROR_FILE_404 . STRING_META);
                $this->setState(ERROR_FILE_404 . STRING_META);
                return (false);
            }

            $this->meta = $tmpMeta;
            unset($tmpMeta);

            if ($this->template->schema != TEMPLATE_DB_MONGO) {
                $this->logger->warn(ERROR_SCHEMA_MISMATCH . $this->template->schema . ERROR_STUB_EXPECTING . TEMPLATE_DB_MONGO);
                $this->setState(ERROR_SCHEMA_MISMATCH . $_template);
                return false;
            }
        } catch (Throwable $t) {
            $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
            $msg = ERROR_THROWABLE_EXCEPTION . COLON . $t->getMessage();
            $this->eventMessages[] = $msg;
            if (isset($this->logger) and $this->logger->available)
                $this->logger->warn($hdr . $msg);
            else
                consoleLog($this->res, CON_ERROR, $hdr . $msg);
            $this->state = STATE_FRAMEWORK_FAIL;
            return (false);
        }
        // transfer meta data info to current instantiation
        $this->useAuditing = $this->template->setAuditing;
        if ($this->useAuditing) {
            try {
                $this->templateAudit = new gatAudit();
            } catch (Throwable $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $this->eventMessages[] = ERROR_THROWABLE_EXCEPTION;
                $this->eventMessages[] = $t->getMessage();
                if ($this->logger->available) $this->logger->warn($hdr . ERROR_THROWABLE_EXCEPTION . $t->getMessage());
                consoleLog($this->res, CON_SYSTEM, $hdr . ERROR_THROWABLE_EXCEPTION . $t->getMessage());
            }
        }
        $this->useJournaling = $this->template->setJournaling;
        if ($this->useJournaling) {
            try {
                $this->templateJournal = new gatJournaling();
            } catch (Throwable $t) {
                $hdr = basename(__METHOD__) . AT . __LINE__ . COLON;
                $this->eventMessages[] = ERROR_THROWABLE_EXCEPTION;
                $this->eventMessages[] = $t->getMessage();
                if ($this->logger->available) $this->logger->warn($hdr . ERROR_THROWABLE_EXCEPTION . $t->getMessage());
                consoleLog($this->res, CON_SYSTEM,  $hdr . ERROR_THROWABLE_EXCEPTION . $t->getMessage());
            }
        }
        $this->dbService = $this->template->service;
        $this->schema = $this->template->schema;
        $this->collectionName = $this->template->collection . $this->template->extension;
        // todo -- view installation goes here... check the mysql deployment for an example
        $this->ext = $this->template->extension;
        $this->useCache = $this->template->setCache;
        $this->useDeletes = $this->template->setDeletes;
        $this->allowUpdates = $this->template->setUpdates;
        $this->useDetailedHistory = $this->template->setHistory;
        $this->defaultStatus = $this->template->setDefaultStatus;
        $this->searchStatus = $this->template->setSearchStatus;
        $this->useLocking = $this->template->setLocking;
        $this->useTimers = ($this->template->setTimers and gasConfig::$settings[CONFIG_DATABASE][CONFIG_DATABASE_QUERY_TIMERS]);
        $this->pKey = $this->template->setPKey;
        $this->useToken = $this->template->setTokens;
        $this->class = $_template;
        $this->debug = gasConfig::$settings[CONFIG_DEBUG];
        $this->cacheExpiry = $this->template->cacheTimer;

        // load sub-collection definitions if they exist
        if (isset($this->template->subC) and is_array($this->template->subC) and !empty($this->template->subC)) {
            foreach ($this->template->subC as $column => $subCollection) {
                if (!empty($subCollection) and is_array($subCollection)) {
                    foreach ($subCollection as $key) {
                        $this->subCollections[($column . $this->ext)][] = ($key . $this->ext);
                    }
                }
            }
        }

        // todo -- evaluate if we want to continue using the hidden-field functionality
        if (isset($this->template->fields) and is_array($this->template->fields)) {
            foreach ($this->template->fields as $key => $value) {
//                if (!in_array($key, $this->hiddenColumns)) {
                    $this->fieldList[] = ($key . $this->ext);
                    $this->fieldTypes[($key . $this->ext)] = $value;
//                }
            }
        }

        if (isset($this->template->protectedFields) and is_array($this->template->protectedFields)) {
            foreach ($this->template->protectedFields as $field) {
                if ($field != MONGO_ID) {
                    $this->protectedFields[] = $field . $this->ext;
                }
            }
        }

        // create the new index list which is the abbreviated col name (sans ext) plus the index labels
        if (isset($this->template->indexFields) and is_array($this->template->indexFields)) {
            foreach ($this->template->indexFields as $key) {
                if (!in_array($key, $this->hiddenColumns)) {
                    $this->indexes[] = ($key . $this->ext);
                    $this->indexList[] = $key;
                }
            }
        }

        // add the index labels to the index list
        if (isset($this->template->indexNameList) and is_array($this->template->indexNameList) and !empty($this->template->indexNameList))
            $this->indexList = array_merge($this->indexList, $this->template->indexNameList);

        // process the indexes
        // currently supported: (singleField, compound, multiKey)
        // currently unsupported: (geoSpatial, test, hashed)

        // single-field:  [fieldName + ext] = sortOrder
        if (isset($this->template->singleFields) and is_array($this->template->singleFields)) {
            foreach ($this->template->singleFields as $key => $value) {
                if (array_key_exists($key, $this->template->fields)) {
                    $this->singleIndexes[($key . $this->ext)] = $value;
                } else {
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_TDE . sprintf(ERROR_MDB_IDX_KEY_404, $key);
                    $this->eventMessages[] = $msg;
                    $this->logger->debug($msg);
                    return(false);
                }
            }
        }

        // compound: [ indexName = [ (fieldName + ext) = sortOrder, ... ] ]
        if (isset($this->template->compoundIndexes) and is_array($this->template->compoundIndexes)) {
            foreach ($this->template->compoundIndexes as $key => $value) {
                foreach ($value as $col => $sortOrder) {
                    if (array_key_exists($key, $this->template->fields)) {
                        $this->compoundIndexes[($key . $this->ext)][($col . $this->ext)] = $sortOrder;
                    } else {
                        $this->compoundIndexes[$key][($col . $this->ext)] = $sortOrder;
                    }
                }
            }
        }

        // multikey: [ indexName = [ (parentFieldName + ext).(childFieldName + ext) = sortOrder, ... ] ]
        // validate that the multiKey index is correctly formatted and that the parent value is an array
        if (isset($this->template->multiKey) and is_array($this->template->multiKey)) {
            foreach ($this->template->multiKey as $key => $value) {
                foreach ($value as $col => $sortOrder) {
                    // split-out the parent array name from the sub-array field name
                    list($pCol, $cCol) = explode(DOT, $col);
                    if ($this->template->fields[$pCol] == DATA_TYPE_ARRAY) {
                        $this->multiKey[$key][($pCol . $this->ext) . DOT . ($cCol . $this->ext)] = $sortOrder;
                    } else {
                        $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                        $msg = $hdr . ERROR_TDE . sprintf(ERROR_MDB_IDX_MULTI_TYPE, $col);
                        $this->eventMessages[] = $msg;
                        $this->logger->debug($msg);
                        return(false);
                    }
                }
            }
        }

        // process the index properties
        // currently supported: (unique, partial, ttl)
        // currently unsupported: (sparse - replaced by partial and no longer recommended)

        // partial: [ < SOME_FIELD_NAME | INDEX-NAME > => [ PARTIAL_INDEX_FIELD_NAME => [ MONGO_OPERATOR => value ]]]
        if (isset($this->template->partialIndexes) and is_array($this->template->partialIndexes)) {
            $count = 0;
            foreach ($this->template->partialIndexes as $indexCount => $partialIndex) {
                // each $partialIndex contains two parts, the query field, and the query proper:
                if (count($partialIndex) != 2) {
                    // record an error message about the malformed index and break out of this loop
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . sprintf(ERROR_DATA_ARRAY_COUNT, 2, INFO_PARTIAL_INDEX, count($partialIndex));
                    $this->eventMessages[] = $msg;
                    $this->logger->error($msg);
                    break;
                }
                $fieldPart = $partialIndex[0];
                $queryPart = $partialIndex[1];
                // first, validate the index name as either a pre-existing label or column name
                if (!in_array(key($fieldPart), $this->template->indexNameList) and !array_key_exists(key($fieldPart), $this->template->fields)) {
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_TDE . sprintf(ERROR_MDB_IDX_LABEL_404, key($fieldPart));
                    $this->eventMessages[] = $msg;
                    if (gasConfig::$settings[CONFIG_DEBUG]) $this->logger->debug($msg);
                    return(false);  // todo -- do we really want to return false?
                } else {
                    $iName = (key($fieldPart) . $this->ext);
                }
                // if we reach this far, we've validated the $fieldPart so now we validate the $queryPart:
                // first, ensure that the container key is MONGO_STRING_PARTIAL_FE:
                if (key($queryPart) != MONGO_STRING_PARTIAL_FE) {
                    // if we don't have this required tag, then error all the things and break the loop
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_PI_TAG_404;
                    $this->eventMessages[] = $msg;
                    $this->logger->error($msg);
                    break;
                }
                // ensure that the query sub-array is actually an array
                $subQuery = $queryPart[MONGO_STRING_PARTIAL_FE];
                if (!is_array($subQuery)) {
                    // if the query part referenced by MONGO_STRING_PARTIAL_FE is not an array, that's an error
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_PI_MALO;
                    $this->eventMessages[] = $msg;
                    $this->logger->error($msg);
                    break;
                }
                // finally, validate that the key is in the current field list; if so, append the extension to the key
                $indexField = key($subQuery);
                if (array_key_exists(key($subQuery), $this->template->fields)) {
                    // the array key for $subQuery must be a member of the template fields
                    $subQuery[($indexField . $this->ext)] = $subQuery[key($subQuery)];
                    unset($subQuery[$indexField]);
                    // overwrite the previously stored query with the json-ready, validated, query array
                    $this->partialIndexes[$count++] = [[ $iName => $fieldPart[ key($fieldPart) ]], [ $subQuery ]];
                } else {
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_TDE . sprintf(ERROR_MDB_IDX_KEY_404, $indexField);
                    $this->eventMessages[] = $msg;
                    $this->logger->debug($msg);
                    break;
                }
            }
        }

        // unique:[ < FIELD_NAME | INDEX-NAME > => <SORT_DIR>, ... ]
        if (isset($this->template->uniqueIndexes) and is_array($this->template->uniqueIndexes)) {
            foreach ($this->template->uniqueIndexes as $col => $sortOrder) {
                if (array_key_exists($col, $this->template->fields)) {
                    $this->uniqueIndexes[($col . $this->ext)] = $sortOrder;
                } else {
                    $this->uniqueIndexes[$col] = $sortOrder;
                }
            }
        }

        // ttl: [ SOME_FIELD_NAME => ExpireVal ]
        if (isset($this->template->ttlIndexes) and is_array($this->template->ttlIndexes)) {
            foreach ($this->template->ttlIndexes as $col => $value) {
                if (array_key_exists($col, $this->template->fields)) {
                    $this->ttlIndexes[($col . $this->ext)] = $value;
                } else {
                    $msg = sprintf(ERROR_MDB_IDX_KEY_404, $col) . ERROR_TDE;
                    $this->eventMessages[] = $msg;
                    $this->logger->debug($msg);
                    return(false);
                }
            }
        }

        if (isset($this->template->regexFields) and is_array($this->template->regexFields)) {
            foreach ($this->template->regexFields as $key) {
                if (in_array(($key . $this->ext), $this->fieldList)) {
                    $this->fuzzyIndexes[] = ($key . $this->ext);
                } else {
                    $hdr = sprintf(INFO_LOC, __METHOD__, __LINE__);
                    $msg = $hdr . ERROR_TDE . ERROR_MDB_IDX_FUZZY_NOT_IDX . ($key . $this->ext);
                    $this->logger->data($msg);
                    $this->eventMessages[] = ERROR_MDB_DIAG_INDEXES;
                    $this->eventMessages[] = $msg;
                    $this->state = STATE_FRAMEWORK_FAIL;
                    $this->status = false;
                    return (false);
                }
            }
        }

        if (!is_null($this->template->cacheMap)) {
            foreach ($this->template->cacheMap as $key => $value) {
                $this->cacheMap[($key . $this->ext)] = $value;
            }
        } else {
            $this->cacheMap = [];
            if (!empty($this->template->exposedFields)) {
                foreach ($this->template->exposedFields as $key) {
                    $this->exposedFields[] = ($key . $this->ext);
                }
            }
        }

        if (!is_null($this->template->binFields)) {
            foreach ($this->template->binFields as $key) {
                $this->binaryFields[] = ($key . $this->ext);
            }
        }

        // CORE-1035: migration data
        if (isset($this->template->migrationMap) and is_array($this->template->migrationMap)) {
            $this->migrationConfig[STRING_MIGRATION_MAP] = $this->template->migrationMap;
        } else {
            $this->migrationConfig[STRING_MIGRATION_MAP] = sprintf(INFO_NOT_SET, STRING_MIGRATION_MAP);
        }
        $this->migrationConfig[STRING_MIGRATION_STATUS_KEY] = (isset($this->template->migrationStatusKV)) ? $this->template->migrationStatusKV : null;
        $this->migrationConfig[STRING_MIGRATION_SORT_KEY] = (isset($this->template->migrationSortKey)) ? $this->template->migrationSortKey : null;

        // warehouse data
        $this->warehouseConfig = (isset($this->template->wareHouse)) ? $this->template->wareHouse : null;

        $this->templateName = $_template;
        if ($this->template->selfDestruct) {
            $this->eventMessages[] = INFO_TEMPLATE_CLASS_DROPPED;
            unset($this->template);
        }
        return (true);
    }


    /**
     * __destruct() -- public function
     *
     * class destructor
     *
     * @author  mike@givingassistant.org
     * @version 1.0
     *
     * HISTORY:
     * ========
     * 07-05-17     mks     original coding
     *
     */
    public function __destruct()
    {
        // As of PHP 5.3.10 destructors are not run on shutdown caused by fatal errors.
        //
        // destructor is registered shut-down function in constructor -- so any recovery
        // efforts should go in this method.

        // there is no destructor method defined in the core abstraction class, hence
        // there is no call to that parent destructor in this class.
        parent::__destruct();
    }
}