[Pki-devel] [PATCH] 632 Fixed cert-find performance.
Endi Sukma Dewata
edewata at redhat.com
Wed Jul 15 00:35:02 UTC 2015
The CertService.searchCerts() has been modified to use the VLV
properly to retrieve just the entries in the requested page, thus
reducing the response time and memory requirement.
Some classes have been modified to clean up the debugging logs.
--
Endi S. Dewata
-------------- next part --------------
From c1846a562d46ea5d5e2eaff846fbe7d742b6a474 Mon Sep 17 00:00:00 2001
From: "Endi S. Dewata" <edewata at redhat.com>
Date: Tue, 14 Jul 2015 19:47:15 -0400
Subject: [PATCH] Fixed cert-find performance.
The CertService.searchCerts() has been modified to use the VLV
properly to retrieve just the entries in the requested page, thus
reducing the response time and memory requirement.
Some classes have been modified to clean up the debugging logs.
---
.../org/dogtagpki/server/ca/rest/CertService.java | 38 +++++-----
.../com/netscape/cms/servlet/cert/ListCerts.java | 86 ++++++++++++++++------
.../cmscore/dbs/CertificateRepository.java | 46 ++++++------
.../com/netscape/cmscore/dbs/DBVirtualList.java | 29 ++++++--
4 files changed, 130 insertions(+), 69 deletions(-)
diff --git a/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java b/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
index e43909bbb59837064711447c5e1733a3ca70970c..440f756dee79904556f9f1671f2638cf22199e22 100644
--- a/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
+++ b/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
@@ -70,6 +70,7 @@ import com.netscape.certsrv.cert.CertSearchRequest;
import com.netscape.certsrv.dbs.EDBRecordNotFoundException;
import com.netscape.certsrv.dbs.certdb.CertId;
import com.netscape.certsrv.dbs.certdb.ICertRecord;
+import com.netscape.certsrv.dbs.certdb.ICertRecordList;
import com.netscape.certsrv.dbs.certdb.ICertificateRepository;
import com.netscape.certsrv.logging.AuditFormat;
import com.netscape.certsrv.logging.ILogger;
@@ -442,47 +443,44 @@ public class CertService extends PKIService implements CertResource {
@Override
public Response searchCerts(CertSearchRequest data, Integer start, Integer size) {
+ CMS.debug("CertService.searchCerts()");
+
if (data == null) {
- throw new BadRequestException("Search request is null.");
+ throw new BadRequestException("Search request is null");
}
start = start == null ? 0 : start;
size = size == null ? DEFAULT_SIZE : size;
String filter = createSearchFilter(data);
- CMS.debug("CertService.searchCerts: filter: " + filter);
+ CMS.debug("CertService: filter: " + filter);
CertDataInfos infos = new CertDataInfos();
try {
- Enumeration<ICertRecord> e = repo.findCertRecords(filter);
- if (e == null) {
- throw new EBaseException("search results are null");
- }
+ ICertRecordList list = repo.findCertRecordsInList(filter, null, "serialno", size);
+ int total = list.getSize();
+ CMS.debug("CertService: total: " + total);
- int i = 0;
+ // return entries in the requested page
+ for (int i = start; i < start + size && i < total; i++) {
+ ICertRecord record = list.getCertRecord(i);
- // skip to the start of the page
- for (; i < start && e.hasMoreElements(); i++)
- e.nextElement();
+ if (record == null) {
+ CMS.debug("CertService: Certificate record not found");
+ throw new PKIException("Certificate record not found");
+ }
- // return entries up to the page size
- for (; i < start + size && e.hasMoreElements(); i++) {
- ICertRecord user = e.nextElement();
- infos.addEntry(createCertDataInfo(user));
+ infos.addEntry(createCertDataInfo(record));
}
- // count the total entries
- for (; e.hasMoreElements(); i++)
- e.nextElement();
-
- infos.setTotal(i);
+ infos.setTotal(total);
if (start > 0) {
URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", Math.max(start - size, 0)).build();
infos.addLink(new Link("prev", uri));
}
- if (start + size < i) {
+ if (start + size < total) {
URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", start + size).build();
infos.addLink(new Link("next", uri));
}
diff --git a/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java b/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
index 185e1fa8e7d24afdde7c4735722609921c6800ea..d0c3fe82a5e9e76288a23ca63f1a51aa7c2c2d92 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
@@ -40,6 +40,7 @@ import netscape.security.x509.X500Name;
import netscape.security.x509.X509CertImpl;
import netscape.security.x509.X509Key;
+import com.netscape.certsrv.apps.CMS;
import com.netscape.certsrv.authentication.IAuthToken;
import com.netscape.certsrv.authorization.AuthzToken;
import com.netscape.certsrv.base.EBaseException;
@@ -352,7 +353,8 @@ public class ListCerts extends CMSServlet {
}
}
- private void processCertFilter(CMSTemplateParams argSet,
+ private void processCertFilter(
+ CMSTemplateParams argSet,
IArgBlock header,
int maxCount,
BigInteger sentinel,
@@ -364,6 +366,14 @@ public class ListCerts extends CMSServlet {
String revokeAll,
Locale locale
) throws EBaseException {
+
+ CMS.debug("ListCerts.processCertFilter()");
+ CMS.debug("ListCerts: max count: " + maxCount);
+ CMS.debug("ListCerts: sentinel: " + sentinel);
+ CMS.debug("ListCerts: total record count: " + totalRecordCount);
+ CMS.debug("ListCerts: serialTo: " + serialTo);
+ CMS.debug("ListCerts: filter: " + filter);
+
BigInteger serialToVal = MINUS_ONE;
try {
@@ -380,6 +390,10 @@ public class ListCerts extends CMSServlet {
} catch (Exception e) {
}
+ CMS.debug("ListCerts: serialToVal: " + serialToVal);
+ CMS.debug("ListCerts: mReverse: " + mReverse);
+ CMS.debug("ListCerts: mHardJumpTo: " + mHardJumpTo);
+
String jumpTo = sentinel.toString();
int pSize = 0;
if (mReverse) {
@@ -390,10 +404,14 @@ public class ListCerts extends CMSServlet {
} else
pSize = maxCount;
+ CMS.debug("ListCerts: pSize: " + pSize);
+
+ CMS.debug("ListCerts: calling findCertRecordsInList() with jumpTo");
ICertRecordList list = mCertDB.findCertRecordsInList(
filter, (String[]) null, jumpTo, mHardJumpTo, "serialno",
pSize);
// retrive maxCount + 1 entries
+ CMS.debug("ListCerts: list size: " + list.getSize());
Enumeration<ICertRecord> e = list.getCertRecords(0, maxCount);
@@ -403,18 +421,24 @@ public class ListCerts extends CMSServlet {
if (!serialToVal.equals(MINUS_ONE)) {
// if user specify a range, we need to
// calculate the totalRecordCount
+ CMS.debug("ListCerts: calling findCertRecordsInList() with serialTo");
tolist = mCertDB.findCertRecordsInList(
filter,
(String[]) null, serialTo,
"serialno", maxCount);
+ CMS.debug("ListCerts: tolist size: " + tolist.getSize());
+
Enumeration<ICertRecord> en = tolist.getCertRecords(0, 0);
if (en == null || (!en.hasMoreElements())) {
+ CMS.debug("ListCerts: no results");
toCurIndex = list.getSize() - 1;
+
} else {
toCurIndex = tolist.getCurrentIndex();
ICertRecord rx = en.nextElement();
BigInteger curToSerial = rx.getSerialNumber();
+ CMS.debug("ListCerts: curToSerial: " + curToSerial);
if (curToSerial.compareTo(serialToVal) == -1) {
toCurIndex = list.getSize() - 1;
@@ -424,11 +448,12 @@ public class ListCerts extends CMSServlet {
}
}
}
+ CMS.debug("ListCerts: toCurIndex: " + toCurIndex);
}
int curIndex = list.getCurrentIndex();
+ CMS.debug("ListCerts: curIndex: " + curIndex);
- int count = 0;
BigInteger firstSerial = new BigInteger("0");
BigInteger curSerial = new BigInteger("0");
ICertRecord[] recs = new ICertRecord[maxCount];
@@ -438,19 +463,22 @@ public class ListCerts extends CMSServlet {
/* in reverse (page up), because the sentinel is the one after the
* last item to be displayed, we need to skip it
*/
+ CMS.debug("ListCerts: records:");
+ int count = 0;
while ((count < ((mReverse && !mHardJumpTo) ? (maxCount + 1) : maxCount)) && e.hasMoreElements()) {
ICertRecord rec = e.nextElement();
if (rec == null) {
- com.netscape.certsrv.apps.CMS.debug("record " + count + " is null");
+ CMS.debug("ListCerts: * record " + count + " is null");
break;
}
curSerial = rec.getSerialNumber();
- com.netscape.certsrv.apps.CMS.debug("record " + count + " is serial#" + curSerial);
+ CMS.debug("ListCerts: * record " + count + ": " + curSerial);
if (count == 0) {
firstSerial = curSerial;
if (mReverse && !mHardJumpTo) {//reverse got one more, skip
+ CMS.debug("ListCerts: skipping record");
count++;
continue;
}
@@ -468,32 +496,34 @@ public class ListCerts extends CMSServlet {
if (!serialToVal.equals(MINUS_ONE)) {
// check if we go over the limit
if (curSerial.compareTo(serialToVal) == 1) {
- com.netscape.certsrv.apps.CMS.debug("curSerial compare serialToVal 1 breaking...");
+ CMS.debug("ListCerts: curSerial compare serialToVal 1 breaking...");
break;
}
}
if (mReverse) {
+ CMS.debug("ListCerts: returning with rcount: " + rcount);
recs[rcount++] = rec;
+
} else {
-
- IArgBlock rarg = com.netscape.certsrv.apps.CMS.createArgBlock();
-
+ CMS.debug("ListCerts: returning with arg block");
+ IArgBlock rarg = CMS.createArgBlock();
fillRecordIntoArg(rec, rarg);
argSet.addRepeatRecord(rarg);
}
+
count++;
}
} else {
- com.netscape.certsrv.apps.CMS.debug(
- "ListCerts::processCertFilter() - no Cert Records found!");
+ CMS.debug("ListCerts: no records found");
return;
}
if (mReverse) {
- // fill records into arg block and argSet
+ CMS.debug("ListCerts: fill records into arg block and argSet");
for (int ii = rcount - 1; ii >= 0; ii--) {
if (recs[ii] != null) {
+ CMS.debug("ListCerts: processing recs[" + ii + "]");
IArgBlock rarg = com.netscape.certsrv.apps.CMS.createArgBlock();
//com.netscape.certsrv.apps.CMS.debug("item "+ii+" is serial # "+ recs[ii].getSerialNumber());
fillRecordIntoArg(recs[ii], rarg);
@@ -507,56 +537,70 @@ public class ListCerts extends CMSServlet {
if (e.hasMoreElements()) {
nextRec = e.nextElement();
+ CMS.debug("ListCerts: next record: " + nextRec.getSerialNumber());
+
+ } else {
+ CMS.debug("ListCerts: no next record");
}
header.addStringValue("op", req.getParameter("op"));
if (revokeAll != null)
header.addStringValue("revokeAll", revokeAll);
+
if (mAuthName != null)
header.addStringValue("issuerName", mAuthName.toString());
+
if (!serialToVal.equals(MINUS_ONE))
header.addStringValue("serialTo", serialToVal.toString());
+
header.addStringValue("serviceURL", req.getRequestURI());
header.addStringValue("queryCertFilter", filter);
header.addStringValue("templateName", "queryCert");
header.addStringValue("queryFilter", filter);
header.addIntegerValue("maxCount", maxCount);
+
if (totalRecordCount == -1) {
if (!serialToVal.equals(MINUS_ONE)) {
totalRecordCount = toCurIndex - curIndex + 1;
- com.netscape.certsrv.apps.CMS.debug("totalRecordCount=" + totalRecordCount);
+ CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
} else {
totalRecordCount = list.getSize() -
list.getCurrentIndex();
- com.netscape.certsrv.apps.CMS.debug("totalRecordCount=" + totalRecordCount);
+ CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
}
}
+ int currentRecordCount = list.getSize() - list.getCurrentIndex();
+ CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
+ CMS.debug("ListCerts: currentRecordCount: " + currentRecordCount);
+
header.addIntegerValue("totalRecordCount", totalRecordCount);
- header.addIntegerValue("currentRecordCount", list.getSize() -
- list.getCurrentIndex());
+ header.addIntegerValue("currentRecordCount", currentRecordCount);
String qs = "";
if (mReverse)
qs = "querySentinelUp";
else
qs = "querySentinelDown";
+ CMS.debug("ListCerts: qs: " + qs);
if (mHardJumpTo) {
- com.netscape.certsrv.apps.CMS.debug("curSerial added to querySentinelUp:" + curSerial.toString());
-
+ CMS.debug("ListCerts: curSerial added to querySentinelUp: " + curSerial);
header.addStringValue("querySentinelUp", curSerial.toString());
+
} else {
if (nextRec == null) {
header.addStringValue(qs, null);
- com.netscape.certsrv.apps.CMS.debug("nextRec is null");
- if (mReverse) {
- com.netscape.certsrv.apps.CMS.debug("curSerial added to querySentinelUp:" + curSerial.toString());
+ CMS.debug("ListCerts: nextRec is null");
+ if (mReverse) {
+ CMS.debug("ListCerts: curSerial added to querySentinelUp: " + curSerial);
header.addStringValue("querySentinelUp", curSerial.toString());
}
+
} else {
BigInteger nextRecNo = nextRec.getSerialNumber();
+ CMS.debug("ListCerts: nextRecNo: " + nextRecNo);
if (serialToVal.equals(MINUS_ONE)) {
header.addStringValue(
@@ -570,7 +614,7 @@ public class ListCerts extends CMSServlet {
null);
}
}
- com.netscape.certsrv.apps.CMS.debug("querySentinel " + qs + " = " + nextRecNo.toString());
+ CMS.debug("ListCerts: querySentinel " + qs + ": " + nextRecNo);
}
} // !mHardJumpto
diff --git a/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java b/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
index 7d626b93d1e4e266c93dc9da31239eeb06133732..8d96265213ad5826a43dfd8ccd8bbf979fd0da4f 100644
--- a/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
+++ b/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
@@ -24,8 +24,8 @@ import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
-import java.util.Vector;
import java.util.Random;
+import java.util.Vector;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
@@ -42,9 +42,9 @@ import netscape.security.x509.X509CertInfo;
import com.netscape.certsrv.apps.CMS;
import com.netscape.certsrv.base.EBaseException;
+import com.netscape.certsrv.base.IConfigStore;
import com.netscape.certsrv.base.MetaInfo;
import com.netscape.certsrv.base.SessionContext;
-import com.netscape.certsrv.base.IConfigStore;
import com.netscape.certsrv.ca.ICRLIssuingPoint;
import com.netscape.certsrv.dbs.EDBException;
import com.netscape.certsrv.dbs.EDBRecordNotFoundException;
@@ -192,7 +192,7 @@ public class CertificateRepository extends Repository
randomNumber = (randomNumber.multiply(mRangeSize)).shiftRight(mBitLength);
CMS.debug("CertificateRepository: getRandomNumber randomNumber="+randomNumber);
- return randomNumber;
+ return randomNumber;
}
private BigInteger getRandomSerialNumber(BigInteger randomNumber) throws EBaseException {
@@ -201,7 +201,7 @@ public class CertificateRepository extends Repository
nextSerialNumber = randomNumber.add(mMinSerialNo);
CMS.debug("CertificateRepository: getRandomSerialNumber nextSerialNumber="+nextSerialNumber);
- return nextSerialNumber;
+ return nextSerialNumber;
}
private BigInteger checkSerialNumbers(BigInteger randomNumber, BigInteger serialNumber) throws EBaseException {
@@ -247,7 +247,7 @@ public class CertificateRepository extends Repository
}
} while (nextSerialNumber == null && i < n);
- return nextSerialNumber;
+ return nextSerialNumber;
}
private Object nextSerialNumberMonitor = new Object();
@@ -295,7 +295,7 @@ public class CertificateRepository extends Repository
}
}
- return nextSerialNumber;
+ return nextSerialNumber;
}
private void updateCounter() {
@@ -363,7 +363,7 @@ public class CertificateRepository extends Repository
}
CMS.debug("CertificateRepository: getInRangeCount count=" + count);
- return count;
+ return count;
}
private BigInteger getInRangeCounter(BigInteger minSerialNo, BigInteger maxSerialNo)
@@ -412,7 +412,7 @@ public class CertificateRepository extends Repository
}
CMS.debug("CertificateRepository: getInRangeCounter: counter=" + counter);
- return counter;
+ return counter;
}
public BigInteger getLastSerialNumberInRange(BigInteger serial_low_bound, BigInteger serial_upper_bound)
@@ -455,7 +455,7 @@ public class CertificateRepository extends Repository
mEnableRandomSerialNumbers = !mEnableRandomSerialNumbers;
mDBConfig.putBoolean(PROP_ENABLE_RANDOM_SERIAL_NUMBERS, mEnableRandomSerialNumbers);
}
- }
+ }
if (mEnableRandomSerialNumbers && mCounter == null) {
mCounter = getInRangeCounter(serial_low_bound, serial_upper_bound);
} else {
@@ -484,7 +484,7 @@ public class CertificateRepository extends Repository
BigInteger ret = new BigInteger(serial_low_bound.toString(10));
- ret = ret.subtract(BigInteger.ONE);
+ ret = ret.subtract(BigInteger.ONE);
CMS.debug("CertificateRepository:getLastCertRecordSerialNo: returning " + ret);
return ret;
}
@@ -523,7 +523,7 @@ public class CertificateRepository extends Repository
BigInteger ret = new BigInteger(serial_low_bound.toString(10));
- ret = ret.subtract(BigInteger.ONE);
+ ret = ret.subtract(BigInteger.ONE);
CMS.debug("CertificateRepository:getLastCertRecordSerialNo: returning " + ret);
if (modeChange && mEnableRandomSerialNumbers) {
@@ -1027,7 +1027,7 @@ public class CertificateRepository extends Repository
} catch (Exception e) {
throw new EBaseException(e.getMessage());
} finally {
- if (s != null)
+ if (s != null)
s.close();
}
return exists;
@@ -1278,21 +1278,25 @@ public class CertificateRepository extends Repository
public ICertRecordList findCertRecordsInList(String filter,
String attrs[], String sortKey, int pageSize)
throws EBaseException {
- IDBSSession s = mDBService.createSession();
- CMS.debug("In findCertRecordsInList");
- CertRecordList list = null;
+ CMS.debug("CertificateRepository.findCertRecordsInList()");
+
+ IDBSSession session = mDBService.createSession();
try {
- IDBVirtualList<ICertRecord> vlist = s.<ICertRecord>createVirtualList(getDN(), filter, attrs,
- sortKey, pageSize);
+ IDBVirtualList<ICertRecord> list = session.<ICertRecord>createVirtualList(
+ getDN(),
+ filter,
+ attrs,
+ sortKey,
+ pageSize);
+
+ return new CertRecordList(list);
- list = new CertRecordList(vlist);
} finally {
- if (s != null)
- s.close();
+ if (session != null)
+ session.close();
}
- return list;
}
public ICertRecordList findCertRecordsInList(String filter,
diff --git a/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java b/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
index 51a9230ac401ed739510889cd213f2b6aede2501..fde67c66304208432d9dad73623778534e54b8c7 100644
--- a/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
+++ b/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
@@ -343,7 +343,11 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
* since you'd better check if the index is out of bound first.
*/
public int getSize() {
+
+ CMS.debug("DBVirtualList.getSize()");
+
if (!mInitialized) {
+
mInitialized = true;
// Do an initial search to get the virtual list size
// Keep one page before and one page after the start
@@ -361,10 +365,13 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
LDAPVirtualListControl cont = null;
if (mJumpTo == null) {
+ CMS.debug("DBVirtualList: searching for entry A");
cont = new LDAPVirtualListControl("A",
mBeforeCount,
mAfterCount);
+
} else {
+ CMS.debug("DBVirtualList: searching for entry " + mJumpTo);
if (mPageSize < 0) {
mBeforeCount = mPageSize * -1;
@@ -374,11 +381,12 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
mBeforeCount,
mAfterCount);
}
+
mPageControls[1] = cont;
getJumpToPage();
}
- CMS.debug("Getting Virtual List size: " + mSize);
+ CMS.debug("DBVirtualList: size: " + mSize);
return mSize;
}
@@ -412,6 +420,9 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
}
private synchronized boolean getEntries() {
+
+ CMS.debug("DBVirtualList.getEntries()");
+
// Specify necessary controls for vlist
// LDAPSearchConstraints cons = mConn.getSearchConstraints();
LDAPSearchConstraints cons = new LDAPSearchConstraints();
@@ -501,7 +512,8 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
//System.out.println( "Returning " + mEntries.size() +
// " entries" );
- CMS.debug("getEntries returning " + mEntries.size());
+ CMS.debug("DBVirtualList: entries: " + mEntries.size());
+
return true;
}
@@ -532,7 +544,7 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
mSelectedIndex = nextCont.getFirstPosition() - 1;
mTop = Math.max(0, mSelectedIndex - mBeforeCount);
- CMS.debug("mTop " + mTop);
+ CMS.debug("DBVirtualList: top: " + mTop);
if (mJumpTo != null) {
mJumpToInitialIndex = mTop;
}
@@ -564,7 +576,9 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
* @param first the index of the first entry of the page you want to fetch
*/
public boolean getPage(int first) {
- CMS.debug("getPage " + first);
+
+ CMS.debug("DBVirtualList.getPage(" + first + ")");
+
if (!mInitialized) {
LDAPVirtualListControl cont = new LDAPVirtualListControl(0,
mBeforeCount,
@@ -658,14 +672,15 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
* the caller should really check the index is within bound before this
* but I'll take care of this just in case they are too irresponsible
*/
- if (!mInitialized)
+ if (!mInitialized) {
mSize = getSize();
+ }
- CMS.debug("getElementAt: " + index + " mTop " + mTop);
+ CMS.debug("DBVirtualList: retrieving entry #" + index);
//System.out.println( "need entry " + index );
if ((index < 0) || (index >= mSize)) {
- CMS.debug("returning null");
+ CMS.debug("DBVirtualList: returning null");
return null;
}
--
1.9.3
More information about the Pki-devel
mailing list