c++ string tokenizer

This is string splitter for STL string. If there is any problem, please contact me. This version is used wstring. You can modify it into "tstring" version for all condition.

 

    void Tokenize(const wstring& str, list<wstring>& tokens, const wstring& delimiters = L" ") {

        // Skip delimiters at beginning.

        wstring::size_type lastPos = str.find_first_not_of(delimiters, 0);

        // Find first "non-delimiter".

        wstring::size_type pos = str.find_first_of(delimiters, lastPos);

        while (wstring::npos != pos || wstring::npos != lastPos) {

            // Found a token, add it to the vector.

            tokens.push_back(str.substr(lastPos, pos - lastPos));

            // Skip delimiters.  Note the "not_of"

            lastPos = str.find_first_not_of(delimiters, pos);

            // Find next "non-delimiter"

            pos = str.find_first_of(delimiters, lastPos);

        }

    }

 

int _tmain(int argc, _TCHAR* argv[]) {

    wstring wstrInput = L"This;Is;A;Test;;;;;;";

    list<wstring> tokens;

    Tokenize(wstrInput, tokens, L";");

    return 0;

}