Cache Connector

Data migrations taking days to complete? Tired of waiting for sluggish systems to service iterations or lookups? Then the Cache Connector may be just what the doctor ordered.

Simply add it to your arsenal, point it at the Connector you already have for reading the data and violá your AL will never be the same. This component was initially built to help in a case where the initial data population of Lotus Connections was taking several days to process around 100K users. This was due to a slow, but vital lookup to a less-than-responsive LDAP directory. We modified the AL to use the Cache Connector (which in turn used the original Connector to fill the cache) and the load completed in 90 minutes instead.

In addition to accelerating lookup operations, the Cache Connector also provides blazing Iterator mode with the data sorted on the specified key Attribute.

To use this scripted Connector, follow the instructions below to copy it into your Config. Then edit three variables in the Config tab script (at the top of the edit window, just after the comments):

connectorName - This the name of the Library Connector that will be used to fill the cache. This Connector must support Iterator mode.
keyAttribute - The name of the input Attribute that will be used as the index to the cache. Note that even if this Attribute has multiple values, only the first one will be used. Furthermore, those entries with no value (or duplicate values) will not be cached.
keyIsCaseSensitive - Determines whether the index based on the keyAttribute is case sensitive or not.

As with the HashMap Connector, simply mark the text below (including the start and end tags) and bring it into your copy buffer. Once copied, right-click on the Connectors folder in the Config Editor and select Paste.

<MetamergeConfig created="Tue Aug 28 13:13:21 CEST 2007" createdBy="NO010196" version="1.2">
<Folder name="AssemblyLines"/>
<Folder name="Connectors">
<Connector name="CacheConnector">
<InheritFrom>system:/Connectors/ibmdi.ScriptConnector</InheritFrom>
<ConnectorMode>Iterator</ConnectorMode>
<ConnectorState>Enabled</ConnectorState>
<Configuration>
<InheritFrom>[parent]</InheritFrom>
<parameter name="connectorType">com.ibm.di.connector.ScriptConnector</parameter>
<parameter name="debug">false</parameter>
<parameter name="includeFiles"/>
<parameter name="parserOption">Useless</parameter>
<parameter name="script"><![CDATA[// Cache Connector v. 1.0
//
// This Connector pre-reads the data references by the Iterator
// named in the connectorName variable into a java.util.HashMap
// and allows searches (lookup). Good for making files randomly
// accessible or sorting them quickly.
//
// Note that all Entries in the Iterator's result set are loaded into
// memory. Note also that this implementation does not support
// duplicates with the same key value -- these are lost.
//

var connectorName = "BuffaloLDAP";  // Connector to use
var keyAttribute = "cn";          // Attribute to provide search key for
var keyIsCaseSensitive = false;      // Controls if search/sort is case sensitive

var tree = new java.util.TreeMap();
var keyset = null;
var itr = null;                   // for the set Iterator

var fileConn = system.loadConnector("Connectors/" + connectorName);

if (fileConn == null)
   throw "Connector specified for caching not found: Connectors/" + connectorName;

function selectEntries()
{
   var e;       // local Entry variable.
   var cnt = 0; // cnt used to ensure uniqueness for the key Attribute

   try { // try-catch used in case connector not initialized
      fileConn.terminate();
   } catch (ignoreException) {  
      // Do nothing
   } 

   main.logmsg("* Initializing Cache Connector using Connectors/" + connectorName);

   fileConn.initialize(null);
   fileConn.selectEntries();

   tree.clear(); // Clear the Hastable.

   try {
      // Load entries from the Iterator into the HashMap
      while ((e = fileConn.getNextEntry()) != null) {
         keyval = e.getString(keyAttribute);

         if (keyval == null)
            break; // skip this one since it has no key value

         if (!keyIsCaseSensitive)
            keyval = keyval.toLowerCase();

         tree.put(keyval, e);
         cnt++;
      }
   } catch (ex) {
      main.logmsg("ERROR", "* Cache Connector - Error occurred during iteration with Connectors/" + 
                     connectorName);
      main.logmsg("ERROR", "* Exception: " + ex);
   }

   // Set up an Iterator for Iterator Mode
   itr = tree.keySet().iterator();
   
   main.logmsg("* " + tree.size() + " of " + cnt + " entries cached.");
   if (cnt != tree.size())
      main.logmsg("WARN", "* Not all data has been cached. Some entries either lacked or had duplicate key values.");
}


function getNextEntry ()
{
   if (!itr.hasNext()) {
      result.setStatus (0);
      return;
   }

   nxt = itr.next();

   entry.merge(tree.get(nxt));
   result.setStatus (1); // Set status to "entry returned"
}


function findEntry ()
{
   var e; // entry
   var crit_vector = search.getCriteria();
   var crit = crit_vector.get(0);
   var cval = crit.value;

   if (!keyIsCaseSensitive)
      cval = cval.toLowerCase();

   // First I have to make sure the data has been cached
   try {
      tree.size();
   } catch (exc) {
      selectEntries(); // If not, then cache it now
   }

   var e = tree.get(cval);

   if (e != null) {
      entry.merge(e);
      result.setStatus (1); // Set status to "entry(s) returned"
   } else 
      result.setStatus (0); // Status = "none found"
}

]]></parameter>
</Configuration>
<ComputeChanges>true</ComputeChanges>
<DeltaBehavior>0</DeltaBehavior>
<DeltaStrict>true</DeltaStrict>
<Parser>
<InheritFrom>[parent]</InheritFrom>
</Parser>
<AttributeMap name="Input">
<InheritFrom>[parent]</InheritFrom>
</AttributeMap>
<AttributeMap name="Output">
<InheritFrom>[parent]</InheritFrom>
</AttributeMap>
<DeltaSettings>
<WhenToCommit>After every database operation</WhenToCommit>
<Driver>CloudScape</Driver>
</DeltaSettings>
<Schema name="Input">
<InheritFrom>[parent]</InheritFrom>
</Schema>
<Schema name="Output">
<InheritFrom>[parent]</InheritFrom>
</Schema>
<LinkCriteria>
<InheritFrom>[parent]</InheritFrom>
<AdvancedLinkMode>false</AdvancedLinkMode>
</LinkCriteria>
<Hooks>
<InheritFrom>[parent]</InheritFrom>
</Hooks>
<CheckpointConfig/>
<SandboxConfig/>
<Reconnect>
<InheritFrom>[parent]</InheritFrom>
<parameter name="initreconnect">false</parameter>
<parameter name="numberOfRetries">1</parameter>
<parameter name="retryDelay">10</parameter>
</Reconnect>
<Operations/>
<OperationCarrierIsProperty>false</OperationCarrierIsProperty>
<PoolDefinition>
<InheritFrom>[parent]</InheritFrom>
<Enabled>false</Enabled>
<MinPoolSize>0</MinPoolSize>
<PurgeInterval>0</PurgeInterval>
<InitializeAttempts>1</InitializeAttempts>
</PoolDefinition>
<PoolInstance/>
<InitializeOption>0</InitializeOption>
</Connector>
</Folder>
<Folder name="Parsers"/>
<Folder name="EventHandlers"/>
<Folder name="Scripts"/>
<JavaLibraries/>
<JavaProperties/>
<Folder name="Includes"/>
<Folder name="Config">
<LogConfig name="Logging"/>
<InstanceProperties name="AutoStart">
<AutoStart/>
</InstanceProperties>
<TombstonesConfig name="Tombstones"/>
<SolutionInterface name="SolutionInterface"/>
</Folder>
<Folder name="Functions"/>
<Folder name="AttributeMaps"/>
<Properties name="Properties">
<Stores/>
</Properties>
</MetamergeConfig>

-- EddieHartman - 28 Aug 2007