[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Pki-devel] [PATCH] 0267-Add-code-to-reindex-data-during-cloning-without-replication



Add code to reindex data during cloning without replication
    
    When setting up a clone, indexes are added before the
    replication agreements are set up and the consumer is initialized.
    Thus, as data is replicated and added to the clone db, the
    data is indexed.
    
    When cloning is done with the replication agreements already set
    up and the data replicated, the existing data is not indexed and
    cannot be accessed in searches.  The data needs to be reindexed.
    
    Related to ticket 1414

Please review,
Ade
From b70e73f359510020ab56257a93e5137f6df6d9a9 Mon Sep 17 00:00:00 2001
From: Ade Lee <alee redhat com>
Date: Wed, 29 Jul 2015 14:23:35 -0400
Subject: [PATCH] Add code to reindex data during cloning without replication

When setting up a clone, indexes are added before the
replication agreements are set up and the consumer is initialized.
Thus, as data is replicated and added to the clone db, the
data is indexed.

When cloning is done with the replication agreements already set
up and the data replicated, the existing data is not indexed and
cannot be accessed in searches.  The data needs to be reindexed.

Related to ticket 1414
---
 base/ca/shared/conf/CS.cfg.in                      |  2 +
 base/ca/shared/conf/indextasks.ldif                | 31 +++++++++
 base/kra/shared/conf/CS.cfg.in                     |  2 +
 base/kra/shared/conf/indextasks.ldif               | 31 +++++++++
 base/ocsp/shared/conf/CS.cfg.in                    |  2 +
 base/ocsp/shared/conf/indextasks.ldif              | 31 +++++++++
 .../cms/servlet/csadmin/ConfigurationUtils.java    | 79 ++++++++++++++--------
 base/tks/shared/conf/CS.cfg.in                     |  2 +
 base/tks/shared/conf/indextasks.ldif               | 31 +++++++++
 base/tps/shared/conf/CS.cfg.in                     |  2 +
 base/tps/shared/conf/indextasks.ldif               | 14 ++++
 11 files changed, 200 insertions(+), 27 deletions(-)
 create mode 100644 base/ca/shared/conf/indextasks.ldif
 create mode 100644 base/kra/shared/conf/indextasks.ldif
 create mode 100644 base/ocsp/shared/conf/indextasks.ldif
 create mode 100644 base/tks/shared/conf/indextasks.ldif
 create mode 100644 base/tps/shared/conf/indextasks.ldif

diff --git a/base/ca/shared/conf/CS.cfg.in b/base/ca/shared/conf/CS.cfg.in
index 3d2dd5e436fa30515e8ececf6283fbd676b27d79..d6642a4067cf11f66252d069c34dea04866b269a 100644
--- a/base/ca/shared/conf/CS.cfg.in
+++ b/base/ca/shared/conf/CS.cfg.in
@@ -828,6 +828,8 @@ preop.internaldb.index_ldif=/usr/share/pki/ca/conf/index.ldif
 preop.internaldb.manager_ldif=/usr/share/pki/server/conf/manager.ldif
 preop.internaldb.post_ldif=/usr/share/pki/ca/conf/vlv.ldif,/usr/share/pki/ca/conf/vlvtasks.ldif
 preop.internaldb.wait_dn=cn=index1160589769, cn=index, cn=tasks, cn=config
+preop.internaldb.index_task_ldif=/usr/share/pki/ca/conf/indextasks.ldif
+preop.internaldb.index_wait_dn=cn=index1160589770,cn=index,cn=tasks,cn=config
 internaldb.multipleSuffix.enable=false
 jobsScheduler._000=##
 jobsScheduler._001=## jobScheduler
diff --git a/base/ca/shared/conf/indextasks.ldif b/base/ca/shared/conf/indextasks.ldif
new file mode 100644
index 0000000000000000000000000000000000000000..0134a7fcd88e591c2ecc298b43f7c97150b71c6e
--- /dev/null
+++ b/base/ca/shared/conf/indextasks.ldif
@@ -0,0 +1,31 @@
+dn: cn=index1160589770, cn=index, cn=tasks, cn=config
+objectclass: top
+objectclass: extensibleObject
+cn: index1160589770
+ttl: 10
+nsinstance: {database}
+nsIndexAttribute "revokedby:eq"
+nsIndexAttribute "issuedby:eq"
+nsIndexAttribute "publicKeyData:eq"
+nsIndexAttribute "clientId:eq"
+nsIndexAttribute "dataType:eq"
+nsIndexAttribute "status:eq"
+nsIndexAttribute "description:eq,pres"
+nsIndexAttribute "serialno:eq,pres"
+nsIndexAttribute "metaInfo:eq,pres"
+nsIndexAttribute "certstatus:eq,pres"
+nsIndexAttribute "requestid:eq,pres"
+nsIndexAttribute "requesttype:eq,pres"
+nsIndexAttribute "requeststate:eq,pres"
+nsIndexAttribute "requestowner:eq,pres"
+nsIndexAttribute "notbefore:eq,pres"
+nsIndexAttribute "notafter:eq,pres"
+nsIndexAttribute "duration:eq,pres"
+nsIndexAttribute "dateOfCreate:eq,pres"
+nsIndexAttribute "revokedOn:eq,pres"
+nsIndexAttribute "archivedBy:eq,pres"
+nsIndexAttribute "ownername:eq,pres,sub"
+nsIndexAttribute "subjectname:eq,pres,sub"
+nsIndexAttribute "requestsourceid:eq,pres,sub"
+nsIndexAttribute "revInfo:eq,pres,sub"
+nsIndexAttribute "extension:eq,pres,sub"
diff --git a/base/kra/shared/conf/CS.cfg.in b/base/kra/shared/conf/CS.cfg.in
index fae77133a2156decda5ca2b0b93000a13136ac16..64a369e0a3822b9284f694c961ba9d178bb7e376 100644
--- a/base/kra/shared/conf/CS.cfg.in
+++ b/base/kra/shared/conf/CS.cfg.in
@@ -236,6 +236,8 @@ preop.internaldb.index_ldif=/usr/share/pki/kra/conf/index.ldif
 preop.internaldb.manager_ldif=/usr/share/pki/server/conf/manager.ldif
 preop.internaldb.post_ldif=/usr/share/pki/kra/conf/vlv.ldif,/usr/share/pki/kra/conf/vlvtasks.ldif
 preop.internaldb.wait_dn=cn=index1160527115, cn=index, cn=tasks, cn=config
+preop.internaldb.index_task_ldif=/usr/share/pki/kra/conf/indextasks.ldif
+preop.internaldb.index_wait_dn=cn=index1160589771,cn=index,cn=tasks,cn=config
 internaldb.multipleSuffix.enable=false
 jobsScheduler._000=##
 jobsScheduler._001=## jobScheduler
diff --git a/base/kra/shared/conf/indextasks.ldif b/base/kra/shared/conf/indextasks.ldif
new file mode 100644
index 0000000000000000000000000000000000000000..f7afa6f43b734e89db5644916a8c371bda444d0d
--- /dev/null
+++ b/base/kra/shared/conf/indextasks.ldif
@@ -0,0 +1,31 @@
+dn: cn=index1160589771, cn=index, cn=tasks, cn=config
+objectclass: top
+objectclass: extensibleObject
+cn: index1160589771
+ttl: 10
+nsinstance: {database}
+nsIndexAttribute "revokedby:eq"
+nsIndexAttribute "issuedby:eq"
+nsIndexAttribute "publicKeyData:eq"
+nsIndexAttribute "clientId:eq"
+nsIndexAttribute "dataType:eq"
+nsIndexAttribute "status:eq"
+nsIndexAttribute "description:eq,pres"
+nsIndexAttribute "serialno:eq,pres"
+nsIndexAttribute "metaInfo:eq,pres"
+nsIndexAttribute "certstatus:eq,pres"
+nsIndexAttribute "requestid:eq,pres"
+nsIndexAttribute "requesttype:eq,pres"
+nsIndexAttribute "requeststate:eq,pres"
+nsIndexAttribute "requestowner:eq,pres"
+nsIndexAttribute "notbefore:eq,pres"
+nsIndexAttribute "notafter:eq,pres"
+nsIndexAttribute "duration:eq,pres"
+nsIndexAttribute "dateOfCreate:eq,pres"
+nsIndexAttribute "revokedOn:eq,pres"
+nsIndexAttribute "archivedBy:eq,pres"
+nsIndexAttribute "ownername:eq,pres,sub"
+nsIndexAttribute "subjectname:eq,pres,sub"
+nsIndexAttribute "requestsourceid:eq,pres,sub"
+nsIndexAttribute "revInfo:eq,pres,sub"
+nsIndexAttribute "extension:eq,pres,sub"
diff --git a/base/ocsp/shared/conf/CS.cfg.in b/base/ocsp/shared/conf/CS.cfg.in
index 9c878e89e8757d8c3be8e3040d54c8c4eb0bf7a7..0cbe20bed853569dc7fc137052a1b307df4967a4 100644
--- a/base/ocsp/shared/conf/CS.cfg.in
+++ b/base/ocsp/shared/conf/CS.cfg.in
@@ -195,6 +195,8 @@ preop.internaldb.index_ldif=/usr/share/pki/ocsp/conf/index.ldif
 preop.internaldb.manager_ldif=/usr/share/pki/server/conf/manager.ldif
 preop.internaldb.post_ldif=
 preop.internaldb.wait_dn=
+preop.internaldb.index_task_ldif=/usr/share/pki/ocsp/conf/indextasks.ldif
+preop.internaldb.index_wait_dn=cn=index1160589772,cn=index,cn=tasks,cn=config
 internaldb.multipleSuffix.enable=false
 jss._000=##
 jss._001=## JSS
diff --git a/base/ocsp/shared/conf/indextasks.ldif b/base/ocsp/shared/conf/indextasks.ldif
new file mode 100644
index 0000000000000000000000000000000000000000..41c3f87426cfaced42cf6ff49277527623a3c26d
--- /dev/null
+++ b/base/ocsp/shared/conf/indextasks.ldif
@@ -0,0 +1,31 @@
+dn: cn=index1160589772, cn=index, cn=tasks, cn=config
+objectclass: top
+objectclass: extensibleObject
+cn: index1160589772
+ttl: 10
+nsinstance: {database}
+nsIndexAttribute "revokedby:eq"
+nsIndexAttribute "issuedby:eq"
+nsIndexAttribute "publicKeyData:eq"
+nsIndexAttribute "clientId:eq"
+nsIndexAttribute "dataType:eq"
+nsIndexAttribute "status:eq"
+nsIndexAttribute "description:eq,pres"
+nsIndexAttribute "serialno:eq,pres"
+nsIndexAttribute "metaInfo:eq,pres"
+nsIndexAttribute "certstatus:eq,pres"
+nsIndexAttribute "requestid:eq,pres"
+nsIndexAttribute "requesttype:eq,pres"
+nsIndexAttribute "requeststate:eq,pres"
+nsIndexAttribute "requestowner:eq,pres"
+nsIndexAttribute "notbefore:eq,pres"
+nsIndexAttribute "notafter:eq,pres"
+nsIndexAttribute "duration:eq,pres"
+nsIndexAttribute "dateOfCreate:eq,pres"
+nsIndexAttribute "revokedOn:eq,pres"
+nsIndexAttribute "archivedBy:eq,pres"
+nsIndexAttribute "ownername:eq,pres,sub"
+nsIndexAttribute "subjectname:eq,pres,sub"
+nsIndexAttribute "requestsourceid:eq,pres,sub"
+nsIndexAttribute "revInfo:eq,pres,sub"
+nsIndexAttribute "extension:eq,pres,sub"
diff --git a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
index c8ab38ce71643ae49e5c14746e7a815cea5b263a..d14c4edefe2f59def3767d101f7f0b4a4ec402fb 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
@@ -88,6 +88,7 @@ import netscape.security.x509.X500Name;
 import netscape.security.x509.X509CertImpl;
 import netscape.security.x509.X509Key;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.velocity.context.Context;
 import org.mozilla.jss.CryptoManager;
 import org.mozilla.jss.CryptoManager.NicknameConflictException;
@@ -1427,6 +1428,11 @@ public class ConfigurationUtils {
 
                     // add the index before replication, add VLV indexes afterwards
                     importLDIFS("preop.internaldb.index_ldif", conn);
+
+                    if (!setupReplication) {
+                        // data has already been replicated but not indexed, reindex here
+                        populateIndexes(conn);
+                    }
                 } else {
                     // data will be replicated from the master to the clone
                     // so clone does not need the data
@@ -1444,6 +1450,51 @@ public class ConfigurationUtils {
         }
     }
 
+    private static void populateIndexes(LDAPConnection conn) throws EPropertyNotFound, IOException, EBaseException {
+        CMS.debug("populateIndexes(): start");
+        IConfigStore cs = CMS.getConfigStore();
+
+        importLDIFS("preop.internaldb.index_task_ldif", conn);
+
+        /* For populating indexes, we need to check if the task has completed.
+           Presence of nsTaskExitCode means task is complete
+         */
+        String wait_dn = cs.getString("preop.internaldb.index_wait_dn", "");
+        if (!StringUtils.isEmpty(wait_dn)) {
+            wait_for_task(conn, wait_dn);
+        }
+    }
+
+    private static void wait_for_task(LDAPConnection conn, String wait_dn) {
+        LDAPEntry task = null;
+        boolean taskComplete = false;
+        CMS.debug("Checking wait_dn " + wait_dn);
+        do {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                // restore the interrupted status
+                Thread.currentThread().interrupt();
+            }
+
+            try {
+                task = conn.read(wait_dn, (String[]) null);
+                if (task != null) {
+                    LDAPAttribute attr = task.getAttribute("nsTaskExitCode");
+                    if (attr != null) {
+                        taskComplete = true;
+                        String val = (String) attr.getStringValues().nextElement();
+                        if (val.compareTo("0") != 0) {
+                            CMS.debug("Error in populating indexes: nsTaskExitCode=" + val);
+                        }
+                    }
+                }
+            } catch (Exception le) {
+                CMS.debug("Still checking wait_dn '" + wait_dn + "' (" + le.toString() + ")");
+            }
+        } while (!taskComplete);
+    }
+
     private static void createBaseEntry(String baseDN, LDAPConnection conn) throws EBaseException {
         try {
             CMS.debug("Creating base DN: " + baseDN);
@@ -1836,33 +1887,7 @@ public class ConfigurationUtils {
              */
             String wait_dn = cs.getString("preop.internaldb.wait_dn", "");
             if (!wait_dn.equals("")) {
-                LDAPEntry task = null;
-                boolean taskComplete = false;
-                CMS.debug("Checking wait_dn " + wait_dn);
-                do {
-                    try {
-                        Thread.sleep(1000);
-                    } catch (InterruptedException e) {
-                        // restore the interrupted status
-                        Thread.currentThread().interrupt();
-                    }
-
-                    try {
-                        task = conn.read(wait_dn, (String[]) null);
-                        if (task != null) {
-                            LDAPAttribute attr = task.getAttribute("nsTaskExitCode");
-                            if (attr != null) {
-                                taskComplete = true;
-                                String val = (String) attr.getStringValues().nextElement();
-                                if (val.compareTo("0") != 0) {
-                                    CMS.debug("Error in populating local VLV indexes: nsTaskExitCode=" + val);
-                                }
-                            }
-                        }
-                    } catch (Exception le) {
-                        CMS.debug("Still checking wait_dn '" + wait_dn + "' (" + le.toString() + ")");
-                    }
-                } while (!taskComplete);
+                wait_for_task(conn, wait_dn);
             }
         } catch (Exception e) {
             CMS.debug("populateVLVIndexes(): Exception thrown: " + e);
diff --git a/base/tks/shared/conf/CS.cfg.in b/base/tks/shared/conf/CS.cfg.in
index f864e298a68653f58a09dacb5a2cdfe21029aa8d..e63f07d134719d1c1326fbe939c12157c0109bad 100644
--- a/base/tks/shared/conf/CS.cfg.in
+++ b/base/tks/shared/conf/CS.cfg.in
@@ -187,6 +187,8 @@ preop.internaldb.index_ldif=/usr/share/pki/tks/conf/index.ldif
 preop.internaldb.manager_ldif=/usr/share/pki/server/conf/manager.ldif
 preop.internaldb.post_ldif=
 preop.internaldb.wait_dn=
+preop.internaldb.index_task_ldif=/usr/share/pki/tks/conf/indextasks.ldif
+preop.internaldb.index_wait_dn=cn=index1160589773,cn=index,cn=tasks,cn=config
 internaldb.multipleSuffix.enable=false
 jss._000=##
 jss._001=## JSS
diff --git a/base/tks/shared/conf/indextasks.ldif b/base/tks/shared/conf/indextasks.ldif
new file mode 100644
index 0000000000000000000000000000000000000000..df8a62bb693c356076e6d5f8a237ac2416fbd683
--- /dev/null
+++ b/base/tks/shared/conf/indextasks.ldif
@@ -0,0 +1,31 @@
+dn: cn=index1160589773, cn=index, cn=tasks, cn=config
+objectclass: top
+objectclass: extensibleObject
+cn: index1160589773
+ttl: 10
+nsinstance: {database}
+nsIndexAttribute "revokedby:eq"
+nsIndexAttribute "issuedby:eq"
+nsIndexAttribute "publicKeyData:eq"
+nsIndexAttribute "clientId:eq"
+nsIndexAttribute "dataType:eq"
+nsIndexAttribute "status:eq"
+nsIndexAttribute "description:eq,pres"
+nsIndexAttribute "serialno:eq,pres"
+nsIndexAttribute "metaInfo:eq,pres"
+nsIndexAttribute "certstatus:eq,pres"
+nsIndexAttribute "requestid:eq,pres"
+nsIndexAttribute "requesttype:eq,pres"
+nsIndexAttribute "requeststate:eq,pres"
+nsIndexAttribute "requestowner:eq,pres"
+nsIndexAttribute "notbefore:eq,pres"
+nsIndexAttribute "notafter:eq,pres"
+nsIndexAttribute "duration:eq,pres"
+nsIndexAttribute "dateOfCreate:eq,pres"
+nsIndexAttribute "revokedOn:eq,pres"
+nsIndexAttribute "archivedBy:eq,pres"
+nsIndexAttribute "ownername:eq,pres,sub"
+nsIndexAttribute "subjectname:eq,pres,sub"
+nsIndexAttribute "requestsourceid:eq,pres,sub"
+nsIndexAttribute "revInfo:eq,pres,sub"
+nsIndexAttribute "extension:eq,pres,sub"
diff --git a/base/tps/shared/conf/CS.cfg.in b/base/tps/shared/conf/CS.cfg.in
index 732d1431951ebf980b7cc841894b7fe8bd039393..7bbeceef60dba8ea368557bdee7d68571123c0b1 100644
--- a/base/tps/shared/conf/CS.cfg.in
+++ b/base/tps/shared/conf/CS.cfg.in
@@ -1629,6 +1629,8 @@ preop.internaldb.manager_ldif=/usr/share/pki/server/conf/manager.ldif
 preop.internaldb.post_ldif=/usr/share/pki/tps/conf/vlv.ldif,/usr/share/pki/tps/conf/vlvtasks.ldif
 preop.internaldb.schema.ldif=/usr/share/pki/server/conf/schema.ldif
 preop.internaldb.wait_dn=cn=index1160528734, cn=index, cn=tasks, cn=config
+preop.internaldb.index_task_ldif=/usr/share/pki/tps/conf/indextasks.ldif
+preop.internaldb.index_wait_dn=cn=index1160589774,cn=index,cn=tasks,cn=config
 preop.module.token=Internal Key Storage Token
 preop.pin=[PKI_RANDOM_NUMBER]
 preop.product.name=CS
diff --git a/base/tps/shared/conf/indextasks.ldif b/base/tps/shared/conf/indextasks.ldif
new file mode 100644
index 0000000000000000000000000000000000000000..1227e2b4b02630f101ba12087bb2ebaf792d5d64
--- /dev/null
+++ b/base/tps/shared/conf/indextasks.ldif
@@ -0,0 +1,14 @@
+dn: cn=index1160589774, cn=index, cn=tasks, cn=config
+objectclass: top
+objectclass: extensibleObject
+cn: index1160589774
+ttl: 10
+nsinstance: {database}
+nsIndexAttribute: "tokenUserID:eq,pres,sub"
+nsIndexAttribute: "tokenID:eq,pres,sub"
+nsIndexAttribute: "dateOfCreate:eq,pres,sub"
+nsIndexAttribute: "dateOfModify:eq,pres,sub"
+nsIndexAttribute: "userCertificate:eq"
+nsIndexAttribute: "tokenSerial:eq"
+nsIndexAttribute: "tokenKeyType:eq"
+nsIndexAttribute: "description:eq,pres"
-- 
2.4.3


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]