[Fedora-directory-commits] ldapserver/ldap/servers/slapd/back-ldbm back-ldbm.h, 1.13, 1.14 index.c, 1.14, 1.15 ldbm_attr.c, 1.9, 1.10 ldbm_index_config.c, 1.7, 1.8

Noriko Hosoi (nhosoi) fedora-directory-commits at redhat.com
Tue Jul 15 16:49:45 UTC 2008


Author: nhosoi

Update of /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv2672/ldap/servers/slapd/back-ldbm

Modified Files:
	back-ldbm.h index.c ldbm_attr.c ldbm_index_config.c 
Log Message:
Resolves: #447353
Summary: RFE: search optimization and single character substring searches
Description: extended the substring key to have 3 types:
* begin (e.g., *^a)
* middle (e.g., *abc)
* end (e.g., *xy$)
* Usage: turn an index object to extensibleobject and set an integer value as
follows:
* dn: cn=sn, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config
* objectClass: extensibleObject
* nsSubStrBegin: 2
* nsSubStrMiddle: 3
* nsSubStrEnd: 2
* [...]



Index: back-ldbm.h
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/back-ldbm.h,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- back-ldbm.h	3 Apr 2008 16:52:47 -0000	1.13
+++ back-ldbm.h	15 Jul 2008 16:49:43 -0000	1.14
@@ -346,6 +346,34 @@
 typedef struct idl_private idl_private;
 typedef struct attrcrypt_private attrcrypt_private;
 
+/*
+ * Special attribute for an index entry 
+ * Usage: turn an index object to extensibleobject and 
+ *        set an integer value for each
+ * dn: cn=sn, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config
+ * objectClass: extensibleObject
+ * nsSubStrBegin: 2
+ * nsSubStrMiddle: 3
+ * nsSubStrEnd: 2
+ * [...]
+ * 
+ * By default, the minimum key length triplets of substring index is 2, 3, 2.
+ * The length is changed by setting this nsSubStrLen value.
+ *
+ * Note: If any of the key length value is modified, the index file needs
+ * to be regenerated.  Otherwise, the index file is going to have mixed
+ * key length.
+ * To change the key length,
+ * 1) stop the server, 2) run db2index -t <attr>, 3) start the server.
+ */
+#define INDEX_ATTR_SUBSTRBEGIN	"nsSubStrBegin"
+#define INDEX_ATTR_SUBSTRMIDDLE	"nsSubStrMiddle"
+#define INDEX_ATTR_SUBSTREND	"nsSubStrEnd"
+
+#define INDEX_SUBSTRBEGIN	0
+#define INDEX_SUBSTRMIDDLE	1
+#define INDEX_SUBSTREND		2
+
 
 /* for the cache of attribute information (which are indexed, etc.) */
 struct attrinfo {
@@ -382,6 +410,10 @@
 											specify an ORDERING matching rule, or the index
 											configuration must define an ORDERING matching rule.
 										 */
+	int	*ai_substr_lens;	/* if the attribute nsSubStrXxx is specivied in
+							 * an index instance (dse.ldif), the substr key 
+							 * len value(s) are stored here.  If not specified, 
+							 * the default length triplet is 2, 3, 2.
 };
 
 #define MAXDBCACHE	20


Index: index.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/index.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -r1.14 -r1.15
--- index.c	20 Jun 2008 15:10:00 -0000	1.14
+++ index.c	15 Jul 2008 16:49:43 -0000	1.15
@@ -1830,8 +1830,13 @@
         Slapi_Value	**esubvals = NULL;
         Slapi_Value	**substresult = NULL;
         Slapi_Value   **origvals = NULL;
-        slapi_call_syntax_values2keys_sv( ai->ai_plugin, vals, &ivals, 
-                                          LDAP_FILTER_SUBSTRINGS );
+		Slapi_PBlock		pipb;
+
+		/* prepare pblock to pass ai_substr_lens */
+		pblock_init( &pipb );
+		slapi_pblock_set( &pipb, SLAPI_SYNTAX_SUBSTRLENS, ai->ai_substr_lens );
+        slapi_call_syntax_values2keys_sv_pb( ai->ai_plugin, vals, &ivals, 
+                                          LDAP_FILTER_SUBSTRINGS, &pipb );
 
         origvals = ivals;
         /* delete only: if the attribute has multiple values,
@@ -1840,8 +1845,8 @@
          * then get rid of them from the being deleted values
          */
         if ( evals != NULL ) {
-            slapi_call_syntax_values2keys_sv( ai->ai_plugin, evals, &esubvals, 
-                                              LDAP_FILTER_SUBSTRINGS );
+            slapi_call_syntax_values2keys_sv_pb( ai->ai_plugin, evals,
+							&esubvals, LDAP_FILTER_SUBSTRINGS, &pipb );
             substresult = valuearray_minus_valuearray( ai->ai_plugin, ivals, esubvals );
             ivals = substresult;
             valuearray_free( &esubvals );


Index: ldbm_attr.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/ldbm_attr.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -r1.9 -r1.10
--- ldbm_attr.c	18 Oct 2007 00:08:34 -0000	1.9
+++ ldbm_attr.c	15 Jul 2008 16:49:43 -0000	1.10
@@ -50,14 +50,6 @@
 attrinfo_new()
 {
     struct attrinfo *p= (struct attrinfo *)slapi_ch_calloc(1, sizeof(struct attrinfo));
-	p->ai_type= 0;
-	p->ai_indexmask= 0;
-	p->ai_plugin= NULL;
-	p->ai_index_rules= NULL;
-	p->ai_dblayer= NULL;
-        p->ai_dblayer_count = 0;
-	p->ai_idl= NULL;
-	p->ai_key_cmp_fn = NULL;
     return p;
 }
 
@@ -156,6 +148,25 @@
 }
 
 void
+_set_attr_substrlen(int index, char *str, int **substrlens)
+{
+	char *p = NULL;
+	/* nsSubStrXxx=<VAL> is passed; 
+	   set it to the attribute's plugin */
+	p = strchr(str, '=');
+	if (NULL != p) {
+		long sublen = strtol(++p, (char **)NULL, 10);
+		if (sublen > 0) {	/* 0 is not acceptable */
+			if (NULL == *substrlens) {
+				*substrlens = (int *)slapi_ch_calloc(1, 
+												sizeof(int) * INDEX_SUBSTRLEN);
+			}
+			(*substrlens)[index] = sublen;
+		}
+	}
+}
+
+void
 attr_index_config(
     backend *be,
     char		*fname,
@@ -172,6 +183,8 @@
 	char	**index_rules = NULL;
 	struct attrinfo	*a;
 	int return_value = -1;
+	char    *p;
+	int *substrlens = NULL;
 
 	attrs = str2charray( argv[0], "," );
 	if ( argc > 1 ) {
@@ -189,34 +202,29 @@
 		attrsyntax_oid = slapi_ch_strdup(plugin_syntax2oid(a->ai_plugin));
 		if ( argc == 1 ) {
 			a->ai_indexmask = (INDEX_PRESENCE | INDEX_EQUALITY |
-			    INDEX_APPROX | INDEX_SUB);
+				INDEX_APPROX | INDEX_SUB);
 		} else {
 			a->ai_indexmask = 0;
 			for ( j = 0; indexes[j] != NULL; j++ ) {
-				if ( strncasecmp( indexes[j], "pres", 4 )
-				    == 0 ) {
+				if ( strncasecmp( indexes[j], "pres", 4 ) == 0 ) {
 					a->ai_indexmask |= INDEX_PRESENCE;
-				} else if ( strncasecmp( indexes[j], "eq", 2 )
-				    == 0 ) {
+				} else if ( strncasecmp( indexes[j], "eq", 2 ) == 0 ) {
 					a->ai_indexmask |= INDEX_EQUALITY;
-				} else if ( strncasecmp( indexes[j], "approx",
-				    6 ) == 0 ) {
+				} else if ( strncasecmp( indexes[j], "approx", 6 ) == 0 ) {
 					a->ai_indexmask |= INDEX_APPROX;
-				} else if ( strncasecmp( indexes[j], "sub", 3 )
-				    == 0 ) {
+				} else if ( strncasecmp( indexes[j], "sub", 3 ) == 0 ) {
 					a->ai_indexmask |= INDEX_SUB;
-				} else if ( strncasecmp( indexes[j], "none", 4 )
-				    == 0 ) {
+				} else if ( strncasecmp( indexes[j], "none", 4 ) == 0 ) {
 					if ( a->ai_indexmask != 0 ) {
 						LDAPDebug(LDAP_DEBUG_ANY,
 							"%s: line %d: index type \"none\" cannot be combined with other types\n",
-						    fname, lineno, 0);
+							fname, lineno, 0);
 					}
 					a->ai_indexmask = INDEX_OFFLINE; /* note that the index isn't available */
 				} else {
 					LDAPDebug(LDAP_DEBUG_ANY,
 						"%s: line %d: unknown index type \"%s\" (ignored)\n",
-					    fname, lineno, indexes[j]);
+						fname, lineno, indexes[j]);
 					LDAPDebug(LDAP_DEBUG_ANY,
 						"valid index types are \"pres\", \"eq\", \"approx\", or \"sub\"\n",
 						0, 0, 0);
@@ -227,53 +235,66 @@
 			j = 0;
 			if (index_rules != NULL) for (; index_rules[j] != NULL; ++j);
 			if (j > 0) { /* there are some candidates */
-			    char** official_rules = (char**)
-			      slapi_ch_malloc ((j + 1) * sizeof (char*));
-			    size_t k = 0;
-			    for (j = 0; index_rules[j] != NULL; ++j) {
-				/* Check that index_rules[j] is an official OID */
-				char* officialOID = NULL;
-				IFP mrINDEX = NULL;
-				Slapi_PBlock* pb = slapi_pblock_new();
-				if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) &&
-				    !slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) &&
-				    !slapi_mr_indexer_create (pb) &&
-				    !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
-				    mrINDEX != NULL &&
-				    !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) &&
-				    officialOID != NULL) {
-				    if (!strcasecmp (index_rules[j], officialOID)) {
-					official_rules[k++] = slapi_ch_strdup (officialOID);
-				    } else {
-					char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
-					LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of \"%s\" (ignored)\n",
-						   preamble, officialOID, index_rules[j] );
-					slapi_ch_free((void**)&preamble);
-				    }
-				} else if (!slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid)) {
-				    LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: "
-					       "unknown or invalid matching rule \"%s\" in index configuration (ignored)\n",
-					       fname, lineno, index_rules[j] );
-				} else { /* assume builtin and use compare fn provided by syntax plugin */
-				    need_compare_fn = 1;
+				char** official_rules =
+						(char**)slapi_ch_malloc ((j + 1) * sizeof (char*));
+				size_t k = 0;
+				for (j = 0; index_rules[j] != NULL; ++j) {
+					/* Check that index_rules[j] is an official OID */
+					char* officialOID = NULL;
+					IFP mrINDEX = NULL;
+					Slapi_PBlock* pb = slapi_pblock_new();
+					if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) &&
+						!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) &&
+						!slapi_mr_indexer_create (pb) &&
+						!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
+						mrINDEX != NULL &&
+						!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) &&
+						officialOID != NULL) {
+						if (!strcasecmp (index_rules[j], officialOID)) {
+						official_rules[k++] = slapi_ch_strdup (officialOID);
+						} else {
+						char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
+						LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of \"%s\" (ignored)\n",
+							   preamble, officialOID, index_rules[j] );
+						slapi_ch_free((void**)&preamble);
+						}
+					} else if (p =
+							   strstr(index_rules[j], INDEX_ATTR_SUBSTRBEGIN)) {
+						_set_attr_substrlen(INDEX_SUBSTRBEGIN, index_rules[j],
+											&substrlens);
+					} else if (p =
+							   strstr(index_rules[j], INDEX_ATTR_SUBSTRMIDDLE)) {
+						_set_attr_substrlen(INDEX_SUBSTRMIDDLE, index_rules[j],
+											&substrlens);
+					} else if (p =
+							   strstr(index_rules[j], INDEX_ATTR_SUBSTREND)) {
+						_set_attr_substrlen(INDEX_SUBSTREND, index_rules[j],
+											&substrlens);
+					} else if (!slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid)) {
+						LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: "
+							   "unknown or invalid matching rule \"%s\" in index configuration (ignored)\n",
+							   fname, lineno, index_rules[j] );
+					} else { /* assume builtin and use compare fn provided by syntax plugin */
+						need_compare_fn = 1;
+					}
+					{/* It would improve speed to save the indexer, for future use.
+						But, for simplicity, we destroy it now: */
+						IFP mrDESTROY = NULL;
+						if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) &&
+						mrDESTROY != NULL) {
+						mrDESTROY (pb);
+						}
+					}
+					slapi_pblock_destroy (pb);
 				}
-				{/* It would improve speed to save the indexer, for future use.
-				    But, for simplicity, we destroy it now: */
-				    IFP mrDESTROY = NULL;
-				    if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) &&
-					mrDESTROY != NULL) {
-					mrDESTROY (pb);
-				    }
+				official_rules[k] = NULL;
+				a->ai_substr_lens = substrlens;
+				if (k > 0) {
+					a->ai_index_rules = official_rules;
+					a->ai_indexmask |= INDEX_RULES;
+				} else {
+					slapi_ch_free((void**)&official_rules);
 				}
-				slapi_pblock_destroy (pb);
-			    }
-			    official_rules[k] = NULL;
-			    if (k > 0) {
-				a->ai_index_rules = official_rules;
-				a->ai_indexmask |= INDEX_RULES;
-			    } else {
-				slapi_ch_free((void**)&official_rules);
-			    }
 			}
 		}
 
@@ -283,7 +304,7 @@
 		if (0 != return_value) {
 			/* fatal error, exit */
 			LDAPDebug(LDAP_DEBUG_ANY,"%s: line %d:Fatal Error: Failed to initialize attribute structure\n",
-			    fname, lineno, 0);
+				fname, lineno, 0);
 			exit( 1 );
 		}
 
@@ -300,24 +321,24 @@
 		if (need_compare_fn) {
 			int rc = plugin_call_syntax_get_compare_fn( a->ai_plugin, &a->ai_key_cmp_fn );
 			if (rc != LDAP_SUCCESS) {
-			    LDAPDebug(LDAP_DEBUG_ANY,
-				      "The attribute [%s] does not have a valid ORDERING matching rule\n",
-				      a->ai_type, 0, 0);
+				LDAPDebug(LDAP_DEBUG_ANY,
+					  "The attribute [%s] does not have a valid ORDERING matching rule\n",
+					  a->ai_type, 0, 0);
 				a->ai_key_cmp_fn = NULL;
 			}
 		}
 
 		if ( avl_insert( &inst->inst_attrs, a, ainfo_cmp, ainfo_dup ) != 0 ) {
 			/* duplicate - existing version updated */
-            attrinfo_delete(&a);
+			attrinfo_delete(&a);
 		}
 	}
 	charray_free( attrs );
 	if ( indexes != NULL ) {
-	    charray_free( indexes );
+		charray_free( indexes );
 	}
 	if ( index_rules != NULL ) {
-	    charray_free( index_rules );
+		charray_free( index_rules );
 	}
 }
 


Index: ldbm_index_config.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/ldbm_index_config.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- ldbm_index_config.c	10 Nov 2006 23:45:39 -0000	1.7
+++ ldbm_index_config.c	15 Jul 2008 16:49:43 -0000	1.8
@@ -122,7 +122,7 @@
 /* used by the two callbacks below, to parse an index entry into something
  * awkward that we can pass to attr_index_config().
  */
-#define MAX_TMPBUF      256
+#define MAX_TMPBUF      1024
 #define ZCAT_SAFE(_buf, _x1, _x2) do { \
     if (strlen(_buf) + strlen(_x1) + strlen(_x2) + 2 < MAX_TMPBUF) { \
         strcat(_buf, _x1); \
@@ -170,18 +170,71 @@
         arglist[argc++] = slapi_ch_strdup(tmpBuf);
     }
 
+    tmpBuf[0] = 0;
     /* Get the list of matching rules from the entry. */
     if (0 == slapi_entry_attr_find(e, "nsMatchingRule", &attr)) {
         for (i = slapi_attr_first_value(attr, &sval); i != -1;
              i = slapi_attr_next_value(attr, i, &sval)) {
             attrValue = slapi_value_get_berval(sval);
             if (0 == i) {
-                tmpBuf[0] = 0;
                 ZCAT_SAFE(tmpBuf, "", attrValue->bv_val);
             } else {
                 ZCAT_SAFE(tmpBuf, ",", attrValue->bv_val);
             }
         }
+    }
+
+    /* Get the substr begin length. note: pick the first value. */
+    if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTRBEGIN, &attr)) {
+        i = slapi_attr_first_value(attr, &sval);
+        if (-1 != i) {
+            attrValue = slapi_value_get_berval(sval);
+            if (0 == tmpBuf[0]) {
+                PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+                            INDEX_ATTR_SUBSTRBEGIN,  attrValue->bv_val);
+            } else {
+                int tmpbuflen = strlen(tmpBuf);
+                char *p = tmpBuf + tmpbuflen;
+                PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+                            INDEX_ATTR_SUBSTRBEGIN,  attrValue->bv_val);
+            }
+        }
+    }
+
+    /* Get the substr middle length. note: pick the first value. */
+    if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTRMIDDLE, &attr)) {
+        i = slapi_attr_first_value(attr, &sval);
+        if (-1 != i) {
+            attrValue = slapi_value_get_berval(sval);
+            if (0 == tmpBuf[0]) {
+                PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+                            INDEX_ATTR_SUBSTRMIDDLE,  attrValue->bv_val);
+            } else {
+                int tmpbuflen = strlen(tmpBuf);
+                char *p = tmpBuf + tmpbuflen;
+                PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+                            INDEX_ATTR_SUBSTRMIDDLE,  attrValue->bv_val);
+            }
+        }
+    }
+
+    /* Get the substr end length. note: pick the first value. */
+    if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTREND, &attr)) {
+        i = slapi_attr_first_value(attr, &sval);
+        if (-1 != i) {
+            attrValue = slapi_value_get_berval(sval);
+            if (0 == tmpBuf[0]) {
+                PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+                            INDEX_ATTR_SUBSTREND,  attrValue->bv_val);
+            } else {
+                int tmpbuflen = strlen(tmpBuf);
+                char *p = tmpBuf + tmpbuflen;
+                PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+                            INDEX_ATTR_SUBSTREND,  attrValue->bv_val);
+            }
+        }
+    }
+    if (0 != tmpBuf[0]) {
         arglist[argc++] = slapi_ch_strdup(tmpBuf);
     }
 




More information about the Fedora-directory-commits mailing list