Question

I need to construct a c++ class that defines a parser. This parser will be able to break up a string into tokens. A token is a distinct string of text – it can be a word, a symbol, or a combination of these. Your parser will eliminate whitespace and break string snippets into individual tokens (where single-character tokens are defined at runtime).

Your parser should behave as follows:

1) Whitespace (spaces, tabs, and new lines) is used only to separate tokens – otherwise, it is ignored.

2) Most tokens are made up of strings of text separated punctuation and other symbols.
3) In addition, single-character tokens (punctuation and/or other symbols) may be defined.

How can I do this by making use of the algoritms module, std::find, std::vector, and iterator types.

The class TokenParser should be declared in TokenParser.h and contain the following methods:

// Default constructor
TokenParser()
Default constructor for this class; creates a parser with no initial single-character tokens.

TokenParser(const vector<char>& tokens)
Constructor for this class; creates a parser with the provided single-character tokens defined.

void setCharacterTokens(const vector<char>& tokens) Resets this parser’s single-character tokens to the provided list of tokens.

void addCharacterTokens(const vector<char>& newTokens)
Adds the provided single-character tokens to the set of tokens this parser recognizes.

void addCharacterToken(char newToken)
Adds the provided single-character token to the set of tokens this parser recognizes.

vector<string> parse(string input) const Parses the string and returns a vector of tokens in string form.

You may, at our option, add any private class methods that help you accomplish this goal. DO NOT add any additional public methods or variables, as this violates encapsulation safety.

It is suggested that students consider using a following private helper method similar to the following:

vector<string> tokenize(string snippet)
dBreaks a whitespace-free snippet into tokens by finding and splitting at single-character tokens.

How can I do this?

Answer 1

//Code Image

//TokenParser.h # include<iostream> #include-string> using namespace std; class TokenParser ( private: vector<char>alldelimiters vector<string> tokenize (string snippet); public: TokenParser ) TokenParser (const vector<char>&tokens) ; void setCharacterTokens (const vector<char>& tokens) void addCharacterTokens (const vector<char>& newTokens) void addCharacterToken (char newToken) vector<string parse (string input) const; //Default constructor of TokenParser TokenParser::TokenParser) //parameterized constructor of TokenParser TokenParser::TokenParser (const vector<char>&tokens) alldelimiterstokens; //Implementation of setcharacterTokens method void TokenParser::setCharacterTokens (const vector<char>& tokens) { alldelimiters = tokens;

//Implementation of addCharacterTokens method void TokenParser: :addCharacterTokens (const vector<char>& newTokens) ( alldelimiters.insert (alldelimiters.end ), newTokens.begin), newTokens.end ) //Implementation of addCharacterToken method void Tokenparser: :addcharacterToken (char newroken) alldelimiters.push_back (newToken) //Implementation of tokenize method vector<string> TokenParser::tokenize (string snippet) t vector<string> parserVector: int position -snippet.find (""); //iterate the loop while (position != -1) { string resultstring = snippet.substr (0, position); snippetsnippet.substr (position 1) parserVector.push back (resultstring) position-snippet.find("") parserVector.push_back (snippet); return parservector

//Implementation of parse method vector<string> TokenParser: :parse (string input) const ( string result -""; int doubleQuotes0; unsigned int j for (j 0 j < input.length j++ [ char character- input[ili if (character== ' ' ll character== '\n' 11 charactert result-result else if (std::find (delimiters.begin), delimiters.end ), character)-delimiters.end )) if (character\") doubleğuotes- doubleQuotes 1; if (doubleQuotes % 2-1) result-result+ characteri result = result + ' '; else t result-result result-result + character; else result-result result-result + character; result-result ' else t result-result + character; vector <string>parser - tokenize (result) return parseri

#include #include #include <iostream> <vector> "TokenParser.h" # include<string> using namespace std; //Implementation of printTokens method void printTokens (const vector<string>& tokens) cout << "" for (auto entry: tokens) I1 IT cout<< entry <<""; cout< "]\n";

int main () const char blocksl "" const char punctuation [.1?'"- const char symbols[] "St-"; - vector<char> blockTokens (begin (blocks), end (blocks)); vector<char> punctuationTokens (begin (punctuation), end (punctuation))i string inputi -"Ioh what a loon ) I am!\tDon't you agree?)"; string input2 = "I'll sing \"Amazing Grace\" for $5.\nPlease call 1 the #867-5309."; TokenParsert parser = new TokenParser(); //Display statements cout << "Input strings|n" In" cout << "In" << "Whitespace only:In" In"; printTokens (parser->parse (inputl)) printTokens (parser->parse (input2))i parser->setcharacterTokens (blockTokens);

cout << "In" <<"Block markers:\n" \n" ; printTokens (parser->parse (inputi)) printTokens (parser->parse (input2)); parser->addCharacterTokens (punctuationTokens); cout << "n" <<"Punctuation added:\n" In" printTokens (parser->parse (inputi)); printTokens (parser- parse (input2)); //Iterate the loop for (char symbol symbols) parser->addCharacterToken (symbol); //Display statement cout << "In" <"Symbols added: \n" printTokens (parser->parse (inputi)) printTokens (parser->parse (input2)); delete parser; return 0i

//Sample Output

Input strings
-------------
[{Oh what a loon() I am! Don't you agree?}]
[I'll sing "Amazing Grace" for $5.
Please call1 the #867-5309.]

Whitespace only:
---------------
[ {Oh what a loon() I am! Don't you agree?} ]
[ I'll sing "Amazing Grace" for $5. Please call1 the #867-5309. ]

Block markers:
--------------
[ { Oh what a loon ( ) I am! Don't you agree? } ]
[ I'll sing " Amazing Grace " for $5. Please call1 the #867-5309. ]

Punctuation added:
------------------
[ { Oh what a loon ( ) I am ! Don ' t you agree ? } ]
[ I ' ll sing " Amazing Grace " for $5 . Please call1 the #867-5309 . ]

Symbols added:
--------------
[ { Oh what a loon ( ) I am ! Don ' t you agree ? } ]
[ I ' ll sing " Amazing Grace " for $ 5 . Please call1 the # 867 - 5309 . ]

//Code to copy

//TokenParser.h

#pragma once

//TokenParser.h

#include<iostream>

#include<string>

using namespace std;

class TokenParser {

private:

     vector<char>delimiters;

     vector<string> tokenize(string snippet) ;

public:

     TokenParser();

     TokenParser(const vector<char>&tokens);

     void setCharacterTokens(const vector<char>& tokens);

     void addCharacterTokens(const vector<char>& newTokens);

     void addCharacterToken(char newToken);

     vector<string>parse(string input) const;

};

//Default constructor of TokenParser

TokenParser::TokenParser() {

}

//parameterized constructor of TokenParser

TokenParser::TokenParser(const vector<char>&tokens) {

     delimiters = tokens;

}

//Implementation of setCharacterTokens method

void TokenParser::setCharacterTokens(const vector<char>& tokens) {

     delimiters = tokens;

}

//Implementation of addCharacterTokens method

void TokenParser::addCharacterTokens(const vector<char>& newTokens) {

     delimiters.insert(delimiters.end(), newTokens.begin(), newTokens.end());

}

//Implementation of addCharacterToken method

void TokenParser::addCharacterToken(char newToken) {

     delimiters.push_back(newToken);

}

//Implementation of tokenize method

vector<string> TokenParser::tokenize(string snippet) {

     vector<string> parserVector;

     int position = snippet.find("");

     //iterate the loop

     while (position != -1) {

          string resultstring = snippet.substr(0, position);

          snippet = snippet.substr(position + 1);

          parserVector.push_back(resultstring);

          position = snippet.find("");

     }

     parserVector.push_back(snippet);

     return parserVector;

    

}

//Implementation of parse method

vector<string> TokenParser::parse(string input) const {

     string result = " ";

     int doubleQuotes = 0;

     unsigned int j;

     for (j = 0; j < input.length(); j++) {

          char character = input[j];

          if (character == ' ' || character == '\n' || character == '\t') {

              result = result + ' ';

          }

          else if (std::find(delimiters.begin(), delimiters.end(), character) != delimiters.end()) {

              if (character == '\"') {

                   doubleQuotes = doubleQuotes + 1;

                   if (doubleQuotes % 2 == 1) {

                        result = result + character;

                        result = result + ' ';

                   }

                   else {

                        result = result + ' ';

                        result = result + character;

                   }

              }else{

                   result = result + ' ';

                   result = result + character;

                   result = result + ' ';

              }

          }

          else {

              result = result + character;

          }   

     }

     vector <string>parser = tokenize(result);

     return parser;

}

//TokenParser.cpp

//TokenParser.cpp

#include <iostream>

#include <vector>

#include "TokenParser.h"

#include<string>

using namespace std;

//Implementation of printTokens method

void printTokens(const vector<string>& tokens)

{

     cout << "[ ";

     for (auto entry : tokens)

     {

          cout << entry << " ";

     }

     cout << "]\n";

}

int main()

{

     const char blocks[] = "\"{}()";

     const char punctuation[] = ".!?'";

     const char symbols[] = "$#-";

     vector<char> blockTokens(begin(blocks), end(blocks));

     vector<char> punctuationTokens(begin(punctuation), end(punctuation));

     string input1 = "{Oh what a loon() I am!\tDon't you agree?}";

     string input2 = "I'll sing \"Amazing Grace\" for $5.\nPlease call1 the #867-5309.";

     TokenParser* parser = new TokenParser();

     //Display statements

     cout << "Input strings\n"

          << "-------------\n"

          << "[" << input1 << "]\n"

          << "[" << input2 << "]\n"

          << "\n";

     cout << "\n"

          << "Whitespace only:\n"

          << "---------------\n";

     printTokens(parser->parse(input1));

     printTokens(parser->parse(input2));

     parser->setCharacterTokens(blockTokens);

     cout << "\n"

          << "Block markers:\n"

          << "--------------\n";

     printTokens(parser->parse(input1));

     printTokens(parser->parse(input2));

     parser->addCharacterTokens(punctuationTokens);

     cout << "\n"

          << "Punctuation added:\n"

          << "------------------\n";

     printTokens(parser->parse(input1));

     printTokens(parser->parse(input2));

     //Iterate the loop

     for (char symbol : symbols)

     {

          parser->addCharacterToken(symbol);

     }

     //Display statement

     cout << "\n"

          << "Symbols added:\n"

          << "--------------\n";

     printTokens(parser->parse(input1));

     printTokens(parser->parse(input2));

     delete parser;

     return 0;

}