efficiently find the first element matching a bit mask

Question

I have a list of N 64-bit integers whose bits represent small sets. Each integer has at most k bits set to 1. Given a bit mask, I would lik

Answer 1

This is the bitwise Kd-tree. It typically needs less than 64 visits per lookup operation. Currently, the selection of the bit (dimension) to pivot on is random.

#include 
#include 
#include 
#include 
#include 

typedef unsigned long long Thing;
typedef unsigned long Number;

unsigned thing_ffs(Thing mask);
Thing rand_mask(unsigned bitcnt);

#define WANT_RANDOM 31
#define WANT_BITS 3

#define BITSPERTHING (CHAR_BIT*sizeof(Thing))
#define NONUMBER ((Number)-1)

struct node {
        Thing value;
        Number num;
        Number nul;
        Number one;
        char pivot;
        } *nodes = NULL;
unsigned nodecount=0;
unsigned itercount=0;

struct node * nodes_read( unsigned *sizp, char *filename);
Number *find_ptr_to_insert(Number *ptr, Thing value, Thing mask);

unsigned grab_matches(Number *result, Number num, Thing mask);
void initialise_stuff(void);

int main (int argc, char **argv)
{
Thing mask;
Number num;
unsigned idx;

srand (time(NULL));
nodes = nodes_read( &nodecount, argv[1]);
fprintf( stdout, "Nodecount=%u\n", nodecount );
initialise_stuff();

#if WANT_RANDOM
mask = nodes[nodecount/2].value | nodes[nodecount/3].value ;
#else
mask = 0x38;
#endif

fprintf( stdout, "\n#### Search mask=%llx\n", (unsigned long long) mask );

itercount = 0;
num = NONUMBER;
idx = grab_matches(&num,0, mask);
fprintf( stdout, "Itercount=%u\n", itercount );

fprintf(stdout, "KdTree search  %16llx\n", (unsigned long long) mask );
fprintf(stdout, "Count=%u Result:\n", idx);
idx = num;
if (idx >= nodecount) idx = nodecount-1;
fprintf( stdout, "num=%4u Value=%16llx\n"
        ,(unsigned) nodes[idx].num
        ,(unsigned long long) nodes[idx].value
        );

fprintf( stdout, "\nLinear search  %16llx\n", (unsigned long long) mask );
for (idx = 0; idx < nodecount; idx++) {
        if ((nodes[idx].value & mask) == nodes[idx].value) break;
        }
fprintf(stdout, "Cnt=%u\n", idx);
if (idx >= nodecount) idx = nodecount-1;
fprintf(stdout, "Num=%4u Value=%16llx\n"
        , (unsigned) nodes[idx].num
        , (unsigned long long) nodes[idx].value );

return 0;
}

void initialise_stuff(void)
{
unsigned num;
Number root, *ptr;
root = 0;

for (num=0; num < nodecount; num++) {
        nodes[num].num = num;
        nodes[num].one = NONUMBER;
        nodes[num].nul = NONUMBER;
        nodes[num].pivot = -1;
        }
nodes[num-1].value = 0; /* last node is guaranteed to match anything */

root = 0;
for (num=1; num < nodecount; num++) {
        ptr = find_ptr_to_insert (&root, nodes[num].value, 0ull );
        if (*ptr == NONUMBER) *ptr = num;
        else fprintf(stderr, "Found %u for %u\n"
                , (unsigned)*ptr, (unsigned) num );
        }
}

Thing rand_mask(unsigned bitcnt)
{struct node * nodes_read( unsigned *sizp, char *filename)
{
struct node *ptr;
unsigned size,used;
FILE *fp;

if (!filename) {
        size = (WANT_RANDOM+0) ? WANT_RANDOM : 9;
        ptr = malloc (size * sizeof *ptr);
#if (!WANT_RANDOM)
        ptr[0].value = 0x0c;
        ptr[1].value = 0x0a;
        ptr[2].value = 0x08;
        ptr[3].value = 0x04;
        ptr[4].value = 0x02;
        ptr[5].value = 0x01;
        ptr[6].value = 0x10;
        ptr[7].value = 0x20;
        ptr[8].value = 0x00;
#else
        for (used=0; used < size; used++) {
                ptr[used].value = rand_mask(WANT_BITS);
                }
#endif /* WANT_RANDOM */
        *sizp = size;
        return ptr;
        }

fp = fopen( filename, "r" );
if (!fp) return NULL;
fscanf(fp,"%u\n",  &size );
fprintf(stderr, "Size=%u\n", size);
ptr = malloc (size * sizeof *ptr);
for (used = 0; used < size; used++) {
        fscanf(fp,"%llu\n",  &ptr[used].value );
        }

fclose( fp );
*sizp = used;
return ptr;
}

Thing value = 0;
unsigned bit, cnt;

for (cnt=0; cnt < bitcnt; cnt++) {
        bit = 54321*rand();
        bit %= BITSPERTHING;
        value |= 1ull << bit;
        }
return value;
}

Number *find_ptr_to_insert(Number *ptr, Thing value, Thing done)
{
Number num=NONUMBER;

while ( *ptr != NONUMBER) {
        Thing wrong;

        num = *ptr;
        wrong = (nodes[num].value ^ value) & ~done;
        if (nodes[num].pivot < 0) { /* This node is terminal */
                /* choose one of the wrong bits for a pivot .
                ** For this bit (nodevalue==1 && searchmask==0 )
                */
                if (!wrong) wrong = ~done ;
                nodes[num].pivot  = thing_ffs( wrong );
                }
        ptr = (wrong & 1ull << nodes[num].pivot) ? &nodes[num].nul : &nodes[num].one;
        /* Once this bit has been tested, it can be masked off. */
        done |= 1ull << nodes[num].pivot ;
        }
return ptr;
}

unsigned grab_matches(Number *result, Number num, Thing mask)
{
Thing wrong;
unsigned count;

for (count=0; num < *result; ) {
        itercount++;
        wrong = nodes[num].value & ~mask;
        if (!wrong) { /* we have a match */
                if (num < *result) { *result = num; count++; }
                /* This is cheap pruning: the break will omit both subtrees from the results.
                ** But because we already have a result, and the subtrees have higher numbers
                ** than our current num, we can ignore them. */
                break;
                }
        if (nodes[num].pivot < 0) { /* This node is terminal */
                break;
                }
        if (mask & 1ull << nodes[num].pivot) {
                /* avoid recursion if there is only one non-empty subtree */
                if (nodes[num].nul >= *result) { num = nodes[num].one; continue; }
                if (nodes[num].one >= *result) { num = nodes[num].nul; continue; }
                count += grab_matches(result, nodes[num].nul, mask);
                count += grab_matches(result, nodes[num].one, mask);
                break;
                }
        mask |= 1ull << nodes[num].pivot;
        num = (wrong & 1ull << nodes[num].pivot) ? nodes[num].nul : nodes[num].one;
        }
return count;
}

unsigned thing_ffs(Thing mask)
{
unsigned bit;

#if 1
if (!mask) return (unsigned)-1;
for ( bit=random() % BITSPERTHING; 1 ; bit += 5, bit %= BITSPERTHING) {
        if (mask & 1ull << bit ) return bit;
        }
#elif 0
for (bit =0; bit < BITSPERTHING; bit++ ) {
        if (mask & 1ull <>=1;
        if (!mask) return bit;
        }
#endif

return 0xffffffff;
}

struct node * nodes_read( unsigned *sizp, char *filename)
{
struct node *ptr;
unsigned size,used;
FILE *fp;

if (!filename) {
        size = (WANT_RANDOM+0) ? WANT_RANDOM : 9;
        ptr = malloc (size * sizeof *ptr);
#if (!WANT_RANDOM)
        ptr[0].value = 0x0c;
        ptr[1].value = 0x0a;
        ptr[2].value = 0x08;
        ptr[3].value = 0x04;
        ptr[4].value = 0x02;
        ptr[5].value = 0x01;
        ptr[6].value = 0x10;
        ptr[7].value = 0x20;
        ptr[8].value = 0x00;
#else
        for (used=0; used < size; used++) {
                ptr[used].value = rand_mask(WANT_BITS);
                }
#endif /* WANT_RANDOM */
        *sizp = size;
        return ptr;
        }

fp = fopen( filename, "r" );
if (!fp) return NULL;
fscanf(fp,"%u\n",  &size );
fprintf(stderr, "Size=%u\n", size);
ptr = malloc (size * sizeof *ptr);
for (used = 0; used < size; used++) {
        fscanf(fp,"%llu\n",  &ptr[used].value );
        }

fclose( fp );
*sizp = used;
return ptr;
}

UPDATE:

I experimented a bit with the pivot-selection, favouring bits with the highest discriminatory value ("information content"). This involves:

• making a histogram of the usage of bits (can be done while initialising)
• while building the tree: choosing the one with frequency closest to 1/2 in the remaining subtrees.

The result: the random pivot selection performed better.