Implementing `strtok` whose delimiter has more than one character

Question

Code snippet:

char str[] = "String1::String2:String3:String4::String5";
char *deli = "::";
char *token = strtok(str,deli);

while(token != NULL)
{
  printf("Token= \"%s\"\n", token);
  token=strtok(NULL,deli);
}

The above code snippet produces the output:

Token="String1"
Token="String2"
Token="String3"
Token="String4"
Token="String5"

but I want the output to be:

Token="String1"
Token="String2:String3:String4"
Token="String5"

I know that I am not getting the expected output because each character in the second argument of strtok is considered as a delimiter.

To get the expected output, I've written a program that uses strstr(and other things) to split the given string into tokens such that I get the expected output. Here is the program:

#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>

int myStrtok(char* str,char* deli)
{
    if(str==NULL || deli==NULL)
        return -1;

    int tokens=0;
    char *token;
    char *output=str;


    while((token=strstr(output,deli))!=NULL)
    {

        bool print=true;

        if(output != token)
        {
            printf("Token = \"");
            tokens++;
            print=false;
        }

        while(output != token)
        {
            putchar(*output);
            output++;
        }

        if(print==false)
            printf("\"\n");
        output+=strlen(deli);
    }

    if(strlen(output)>0)
    {
        printf("Token = \"%s\"",output);
        tokens++;
    }
    printf("\n\n");
    return tokens;
}

int main(void)
{
    char str[]="One:1:Two::Three::::";
    char *deli="::";

    int retval;
    printf("Original string=\"%s\"\n\n",str);

    if((retval=myStrtok(str,deli))==-1)
        printf("The string or the delimeter is NULL\n");
    else
        printf("Number of tokens=%d\n", retval);
    return(EXIT_SUCCESS);
}

The above program produces the expected output.

I'm wondering if there are any easier/simpler ways to do it. Are there any?

Answer 1

A string-delimiter function that uses strtok's prototype and mimicks its usage:

char *strtokm(char *str, const char *delim)
{
    static char *tok;
    static char *next;
    char *m;

    if (delim == NULL) return NULL;

    tok = (str) ? str : next;
    if (tok == NULL) return NULL;

    m = strstr(tok, delim);

    if (m) {
        next = m + strlen(delim);
        *m = '\0';
    } else {
        next = NULL;
    }

    return tok;
}

Answer 2

If you don't care about the same usage as strtok I would go with this:

// "String1::String2:String3:String4::String5" with delimiter "::" will produce
// "String1\0\0String2:String3:String4\0\0String5"
// And words should contain a pointer to the first S, the second S and the last S.
char **strToWordArray(char *str, const char *delimiter)
{
  char **words;
  int nwords = countWords(str, delimiter); //I let you decide how you want to do this
  words = malloc(sizeof(*words) * (nwords + 1));

  int w = 0;
  int len = strlen(delimiter);
  words[w++] = str;
  while (*str != NULL)
  {
    if (strncmp(str, delimiter, len) == 0)
    {
      for (int i = 0; i < len; i++)
      {
        *(str++) = 0;
      }
      if (*str != 0)
        words[w++] = str;
      else
        str--; //Anticipate wrong str++ down;
    }
    str++;
  }
  words[w] = NULL;
  return words;
}

Answer 3

code derived from strsep https://code.woboq.org/userspace/glibc/string/strsep.c.html

char *strsepm( char **stringp, const char *delim ) {

    char *begin, *end;

    begin = *stringp;

    if  ( begin == NULL ) return NULL;

    /* Find the end of the token.  */
    end = strstr( begin , delim );

    if ( end != NULL ) {

        /* Terminate the token and set *STRINGP past NUL character.  */
        *end = '\0';

        end  += strlen( delim );

        *stringp = end;

    } else {

        /* No more delimiters; this is the last token.  */
        *stringp = NULL;  
    }

    return begin;
}

int main( int argc , char *argv [] ) {

    char            *token_ptr;
    char            *token;
    const char      *delimiter = "&&";

    char            buffer [ 256 ];

    strcpy( buffer , " && Hello && Bernd && waht's && going && on &&");

    token_ptr = buffer;

    while ( ( token = strsepm( &token_ptr , delimiter ) ) != NULL ) {

        printf( "\'%s\'\n" , token );

    }
}

Result:

' '   
' Hello '    
' Bernd '    
' waht's '    
' going '    
' on '    
''