Open asmwarrior opened 1 week ago
My apologies for the errors, I fixed them, thanks to Code::Blocks.
I never thought that in 2024, Microsoft would continue the 'embrace & extend' strategy.
On Fri, Oct 11, 2024 at 11:38 AM ollydbg @.***> wrote:
I got a lot of error, see below when using GCC compiler:
-------------- Build: Debug in test_parser_lib (compiler: GNU GCC Compiler)---------------
[ 50.0%] g++.exe -Wall -fexceptions -g -Iinclude -c main.cpp -o obj\Debug\main.o [100.0%] g++.exe -o bin\Debug\test_parser_lib.exe obj\Debug\main.o In file included from include/parserlib/core.hpp:5, from include/parserlib.hpp:5, from main.cpp:2: include/parserlib/core/TerminalParser.hpp:22:22: error: declaration of 'typedef Char parserlib::core::TerminalParser
::Char' shadows template parameter 22 | typedef Char Char; | ^~~~ include/parserlib/core/TerminalParser.hpp:16:15: note: template parameter 'Char' declared here 16 | template | ^~~~~ In file included from include/parserlib/core/ParseContext.hpp:11, from include/parserlib/core/Rule.hpp:7, from include/parserlib/core.hpp:10: include/parserlib/core/ParseState.hpp:18:24: error: declaration of 'typedef Source parserlib::core::ParseState I just copied the CFE example code to my "main.cpp".
Any ideas?
I guess you are using MSVC compiler, which is more permissive?
— Reply to this email directly, view it on GitHub https://github.com/axilmar/parserlib/issues/16, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAESDFEUDIN35LYJZ2JZQBDZ26FA3AVCNFSM6AAAAABPYODGFOVHI2DSMVQWIX3LMV43ASLTON2WKOZSGU4DANZZGQ4DEMA . You are receiving this because you are subscribed to this thread.Message ID: @.***>
OK, thanks for the fix!
#include "parserlib.hpp"
#include <iostream>
using namespace parserlib;
using namespace core;
using namespace cfe;
class CalculatorCFE {
public:
enum class TokenId {
Number,
AddOp,
SubOp,
MulOp,
DivOp,
LeftParen,
RightParen
};
enum class ASTId {
Number,
AddExpr,
SubExpr,
MulExpr,
DivExpr
};
template <class CharT, class CharTraits>
friend std::basic_ostream<CharT, CharTraits>& operator << (std::basic_ostream<CharT, CharTraits>& stream, ASTId id) {
switch (id) {
case ASTId::Number:
stream << "Number";
break;
case ASTId::AddExpr:
stream << "Add";
break;
case ASTId::SubExpr:
stream << "Sub";
break;
case ASTId::MulExpr:
stream << "Mul";
break;
case ASTId::DivExpr:
stream << "Div";
break;
}
return stream;
}
typedef CFE<TokenId, ASTId> CFE;
typedef typename CFE::TokenizerRule TokenizerRule;
typedef typename CFE::ParserRule ParserRule;
typedef typename CFE::ASTPtr ASTPtr;
typedef typename CFE::ASTContainer ASTContainer;
typedef typename CFE::ErrorContainer ErrorContainer;
typedef typename CFE::Source Source;
typedef typename CFE::TokenContainerPtr TokenContainerPtr;
typedef typename CFE::ASTContainerPtr ASTContainerPtr;
typedef typename CFE::ErrorContainerPtr ErrorContainerPtr;
CalculatorCFE() {
/**** tokenizer ****/
auto ws = oneIn('\0', ' ');
auto digit = oneIn('0', '9');
auto sign = oneOf('+', '-');
auto integer = +digit;
auto number = (-sign >> integer >> -('.' >> integer))->*TokenId::Number;
auto addOp = term('+')->*TokenId::AddOp;
auto subOp = term('-')->*TokenId::SubOp;
auto mulOp = term('*')->*TokenId::MulOp;
auto divOp = term('/')->*TokenId::DivOp;
auto leftParen = term('(')->*TokenId::LeftParen;
auto rightParen = term(')')->*TokenId::RightParen;
m_tokenizerGrammar = *(ws | number | addOp | subOp | mulOp | divOp | leftParen | rightParen);
/**** parser ****/
auto parenExpr = TokenId::LeftParen >> m_add >> TokenId::RightParen;
auto num = term(TokenId::Number)->*ASTId::Number;
auto val
= parenExpr
| num;
m_mul
= (m_mul >> TokenId::MulOp >> val)->*ASTId::MulExpr
| (m_mul >> TokenId::DivOp >> val)->*ASTId::DivExpr
| val;
m_add
= (m_add >> TokenId::AddOp >> m_mul)->*ASTId::AddExpr
| (m_add >> TokenId::SubOp >> m_mul)->*ASTId::SubExpr
| m_mul;
m_parserGrammar = m_add;
}
std::tuple<bool, TokenContainerPtr, ASTContainerPtr, ErrorContainerPtr> parse(Source& input) {
return CFE::parse(input, m_tokenizerGrammar, m_parserGrammar);
}
static double evaluate(const ASTPtr& ast) {
switch (ast->getID()) {
case ASTId::Number: {
std::stringstream stream;
stream << ast->getSource();
double r;
stream >> r;
return r;
}
case ASTId::AddExpr:
return evaluate(ast->getChildren()[0]) + evaluate(ast->getChildren()[1]);
case ASTId::SubExpr:
return evaluate(ast->getChildren()[0]) - evaluate(ast->getChildren()[1]);
case ASTId::MulExpr:
return evaluate(ast->getChildren()[0]) * evaluate(ast->getChildren()[1]);
case ASTId::DivExpr:
return evaluate(ast->getChildren()[0]) / evaluate(ast->getChildren()[1]);
}
throw std::logic_error("invalid CalculatorCFE ASTId.");
}
private:
TokenizerRule m_tokenizerGrammar;
ParserRule m_add;
ParserRule m_mul;
ParserRule m_parserGrammar;
};
int main()
{
CalculatorCFE calc;
SourceString input = "1.5 + 8.9";
auto [success, tokens, ast, errors] = calc.parse(input);
std::cout << CalculatorCFE::evaluate((*ast)[0]) << std::endl;
return 0;
}
The above code works correctly, and the result is:
10.4
BTW, I see you have comment out a lot of the code in the file tests/parserlib_cfe_tests.cpp
, is that by design?
I commented it out because I hit a wall: the 32-bit MSVC run out of memory and I was trying to fix that.
But I didn't manage to solve the issue, so I installed the 64-bit MSVC, along with Code::Blocks and mingw64, and now I am fully 64-bit and I have put the tests back.
On Sat, Oct 12, 2024 at 6:18 AM ollydbg @.***> wrote:
OK, thanks for the fix!
include "parserlib.hpp"
include
using namespace parserlib; using namespace core; using namespace cfe;
class CalculatorCFE { public: enum class TokenId { Number, AddOp, SubOp, MulOp, DivOp, LeftParen, RightParen };
enum class ASTId { Number, AddExpr, SubExpr, MulExpr, DivExpr }; template <class CharT, class CharTraits> friend std::basic_ostream<CharT, CharTraits>& operator << (std::basic_ostream<CharT, CharTraits>& stream, ASTId id) { switch (id) { case ASTId::Number: stream << "Number"; break; case ASTId::AddExpr: stream << "Add"; break; case ASTId::SubExpr: stream << "Sub"; break; case ASTId::MulExpr: stream << "Mul"; break; case ASTId::DivExpr: stream << "Div"; break; } return stream; } typedef CFE<TokenId, ASTId> CFE; typedef typename CFE::TokenizerRule TokenizerRule; typedef typename CFE::ParserRule ParserRule; typedef typename CFE::ASTPtr ASTPtr; typedef typename CFE::ASTContainer ASTContainer; typedef typename CFE::ErrorContainer ErrorContainer; typedef typename CFE::Source Source; typedef typename CFE::TokenContainerPtr TokenContainerPtr; typedef typename CFE::ASTContainerPtr ASTContainerPtr; typedef typename CFE::ErrorContainerPtr ErrorContainerPtr; CalculatorCFE() { /**** tokenizer ****/ auto ws = oneIn('\0', ' '); auto digit = oneIn('0', '9'); auto sign = oneOf('+', '-'); auto integer = +digit; auto number = (-sign >> integer >> -('.' >> integer))->*TokenId::Number; auto addOp = term('+')->*TokenId::AddOp; auto subOp = term('-')->*TokenId::SubOp; auto mulOp = term('*')->*TokenId::MulOp; auto divOp = term('/')->*TokenId::DivOp; auto leftParen = term('(')->*TokenId::LeftParen; auto rightParen = term(')')->*TokenId::RightParen; m_tokenizerGrammar = *(ws | number | addOp | subOp | mulOp | divOp | leftParen | rightParen); /**** parser ****/ auto parenExpr = TokenId::LeftParen >> m_add >> TokenId::RightParen; auto num = term(TokenId::Number)->*ASTId::Number; auto val = parenExpr | num; m_mul = (m_mul >> TokenId::MulOp >> val)->*ASTId::MulExpr | (m_mul >> TokenId::DivOp >> val)->*ASTId::DivExpr | val; m_add = (m_add >> TokenId::AddOp >> m_mul)->*ASTId::AddExpr | (m_add >> TokenId::SubOp >> m_mul)->*ASTId::SubExpr | m_mul; m_parserGrammar = m_add; } std::tuple<bool, TokenContainerPtr, ASTContainerPtr, ErrorContainerPtr> parse(Source& input) { return CFE::parse(input, m_tokenizerGrammar, m_parserGrammar); } static double evaluate(const ASTPtr& ast) { switch (ast->getID()) { case ASTId::Number: { std::stringstream stream; stream << ast->getSource(); double r; stream >> r; return r; } case ASTId::AddExpr: return evaluate(ast->getChildren()[0]) + evaluate(ast->getChildren()[1]); case ASTId::SubExpr: return evaluate(ast->getChildren()[0]) - evaluate(ast->getChildren()[1]); case ASTId::MulExpr: return evaluate(ast->getChildren()[0]) * evaluate(ast->getChildren()[1]); case ASTId::DivExpr: return evaluate(ast->getChildren()[0]) / evaluate(ast->getChildren()[1]); } throw std::logic_error("invalid CalculatorCFE ASTId."); }
private: TokenizerRule m_tokenizerGrammar; ParserRule m_add; ParserRule m_mul; ParserRule m_parserGrammar; };
int main() { CalculatorCFE calc; SourceString input = "1.5 + 8.9"; auto [success, tokens, ast, errors] = calc.parse(input);
std::cout << CalculatorCFE::evaluate((*ast)[0]) << std::endl; return 0;
}
The above code works correctly, and the result is:
10.4
BTW, I see you have comment out a lot of the code in the file tests/parserlib_cfe_tests.cpp, is that by design?
— Reply to this email directly, view it on GitHub https://github.com/axilmar/parserlib/issues/16#issuecomment-2408328485, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAESDFAFDKHDX2PHFOUN4Y3Z3CIINAVCNFSM6AAAAABPYODGFOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDIMBYGMZDQNBYGU . You are receiving this because you commented.Message ID: @.***>
I commented it out because I hit a wall: the 32-bit MSVC run out of memory and I was trying to fix that. But I didn't manage to solve the issue, so I installed the 64-bit MSVC, along with Code::Blocks and mingw64, and now I am fully 64-bit and I have put the tests back.
Nice work! Thanks. Hope you will solve the 32-bit MSVC issue soon.
I see another issue:
In the testing code of a standalone parser, there are some code like:
struct Token {
TokenKind kind;
std::string lexeme;
int row;
int column;
bool operator == (TokenKind tk) const {
return kind == tk;
}
};
std::vector<Token> tokens;
And in the library's header file, there are same class Token
, see here:
namespace parserlib::cfe {
/**
* A token.
*
* It is created from tokenizing an input, and it is fed to a parser to parse a language grammar.
*
* @param TokenID_ id of token.
*
* @param Source_ source for the token.
*/
template <class TokenID_ = int, class Source_ = core::SourceString<>> class Token {
public:
/**
* Token id type.
*/
typedef TokenID_ TokenID;
/**
* Source type.
*/
typedef Source_ Source;
/**
* Source iterator type.
*/
typedef typename Source::const_iterator SourceIterator;
/**
* Source view type.
*/
typedef std::basic_string_view<typename Source::value_type> StringView;
/**
* Constructor.
* @param id id of token.
* @param startPosition start position of the token into the source.
* @param endPosition end position of the the token into the source.
*/
Token(const TokenID& id, const SourceIterator& startPosition, const SourceIterator& endPosition)
: m_id(id)
, m_startPosition(startPosition)
, m_endPosition(endPosition)
{
}
/**
* Returns the token id.
* @return the token id.
*/
const TokenID& getID() const {
return m_id;
}
/**
* Returns the start position.
* @return the start position.
*/
const SourceIterator& getStartPosition() const {
return m_startPosition;
}
/**
* Returns the end position.
* @return the end position.
*/
const SourceIterator& getEndPosition() const {
return m_endPosition;
}
/**
* Returns the source that correponds to this token.
* @return the source that correponds to this token.
*/
StringView getSource(size_t maxChars = -1) const {
const auto* src = &*m_startPosition;
const size_t size = std::min(maxChars, (size_t)(m_endPosition - m_startPosition));
return StringView(src, size);
}
/**
* Checks if this token has the given id.
* @param id id to check.
* @return true if this token has the given id, false otherwise.
*/
bool operator == (const TokenID& id) const {
return m_id == id;
}
/**
* Checks if this token does not have the given id.
* @param id id to check.
* @return true if this token does not have the given id, false otherwise.
*/
bool operator != (const TokenID& id) const {
return m_id != id;
}
private:
TokenID m_id;
SourceIterator m_startPosition;
SourceIterator m_endPosition;
};
So the names will get conflict if you use the
using namespace parserlib::cfe;
I try to make a standalone parser code, which can parse something like "A=B;".
see below:
#include "parserlib.hpp"
#include <iostream>
#include <vector>
#include <string>
#include <memory>
#include <cassert>
// using namespace parserlib;
using namespace parserlib::core;
// using namespace parserlib::cfe;
enum class TokenKind {
A,
B,
C,
Equal,
Semicolumn
};
struct Token {
TokenKind kind;
std::string lexeme;
int row;
int column;
bool operator == (TokenKind tk) const {
return kind == tk;
}
};
std::vector<Token> tokens;
enum class ASTID {
Assignment,
Declaration
};
// try to parse the "A = B;" statement
// try to parse the "A B;" statement
const auto grammar
= *(TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Assignment
| *(TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Declaration
;
class MyAST {
public:
typedef ASTID ASTID;
typedef std::vector<Token> Source;
ASTID ID;
std::vector<std::shared_ptr<MyAST>> children;
MyAST(ASTID id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end)
: ID(id)
{
}
void addChild(const std::shared_ptr<MyAST>& child) {
children.push_back(child);
}
// Recursive function to print the AST tree
void printTree(int depth = 0) const {
// Print the current node with indentation based on the depth
std::string indent(depth * 2, ' ');
std::cout << indent << "Node: " << astIDToString(ID) << std::endl;
// Recursively print children
for (const auto& child : children) {
child->printTree(depth + 1);
}
}
// Assignment,
// Declaration
// Helper function to convert ASTID enum to string for printing
std::string astIDToString(ASTID id) const {
switch (id) {
case ASTID::Assignment: return "Assignment";
case ASTID::Declaration: return "Declaration";
default: return "Unknown";
}
}
};
std::vector<std::shared_ptr<MyAST>> ast;
class Error {
public:
std::vector<Token>::const_iterator START;
Error(int id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end)
: START(start)
{
}
bool operator < (const Error& err) const {
return START < err.START;
}
};
int main()
{
std::vector<Error> errors;
std::vector<Token> input;
input.push_back(Token{ TokenKind::A, "", 0, 0 });
input.push_back(Token{ TokenKind::Equal, "", 0, 0 });
input.push_back(Token{ TokenKind::B, "", 0, 0 });
input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 });
input.push_back(Token{ TokenKind::A, "", 0, 0 });
input.push_back(Token{ TokenKind::B, "", 0, 0 });
input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 });
parserlib::cfe::parse(input, grammar, ast, errors);
// Draw the AST tree in the console
std::cout << "AST Tree:" << std::endl;
for (const auto& node : ast) {
node->printTree(); // Print the AST tree starting from each root node
}
return 0;
}
I got the build error:
-------------- Build: Debug in test_parser_lib (compiler: GNU GCC Compiler)---------------
[ 50.0%] g++.exe -Wall -fexceptions -g -Iinclude -c main.cpp -o obj\Debug\main.o
[100.0%] g++.exe -o bin\Debug\test_parser_lib.exe obj\Debug\main.o
main.cpp:42:24: error: no match for 'operator>>' (operand types are 'TokenKind' and 'TokenKind')
42 | = *(TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Assignment
| ~~~~~~~~~~~~ ^~ ~~~~~~~~~~~~~~~~
| | |
| TokenKind TokenKind
main.cpp:43:24: error: no match for 'operator>>' (operand types are 'TokenKind' and 'TokenKind')
43 | | *(TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Declaration
| ~~~~~~~~~~~~ ^~ ~~~~~~~~~~~~
| | |
| TokenKind TokenKind
Process terminated with status 1 (0 minute(s), 0 second(s))
2 error(s), 0 warning(s) (0 minute(s), 0 second(s))
So, the error line comes from here:
// try to parse the "A = B;" statement
// try to parse the "A B;" statement
const auto grammar
= *(TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Assignment
| *(TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) ->* ASTID::Declaration
;
Can you help to solve this issue? It looks like the TokenKind can not be used as a "term"?
But in your example code:
const auto grammar
= *(TokenKind::A->*ASTID::A
| TokenKind::B->*ASTID::B
| TokenKind::C->*ASTID::C)
;
This works OK.
Thanks.
Oh, the grammar could be:
// try to parse the "A = B;" statement
// try to parse the "A B;" statement
const auto grammar
= *(TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn ->* ASTID::Assignment)
| *(TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn ->* ASTID::Declaration)
;
See the parenthesis position changes, but still the same build error.
const auto grammar
= *(TokenKind::A >> TokenKind::Equal ->*ASTID::Assignment
| TokenKind::C->*ASTID::Declaration)
;
The above code compiles correctly without error.
But the below code has build errors:
const auto grammar
= *(TokenKind::A >> TokenKind::Equal >> TokenKind::B ->*ASTID::Assignment
| TokenKind::C->*ASTID::Declaration)
;
I'm not sure why.
The former has one >>
and the later has two >>
operator.
The following code:
TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn
does not contain anything from parserlib, and so the compiler thinks it's a declaration of right shifts.
You should put at least one term call in that, like this:
term(TokenKind::A) >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolon
On Thu, Oct 17, 2024 at 11:57 AM ollydbg @.***> wrote:
I try to make a standalone parser code, which can parse something like "A=B;".
see below:
include "parserlib.hpp"
include
include
include
include
include
// using namespace parserlib; using namespace parserlib::core; // using namespace parserlib::cfe;
enum class TokenKind { A, B, C, Equal, Semicolumn };
struct Token { TokenKind kind; std::string lexeme; int row; int column;
bool operator == (TokenKind tk) const { return kind == tk; }
};
std::vector
tokens; enum class ASTID { Assignment, Declaration };
// try to parse the "A = B;" statement // try to parse the "A B;" statement const auto grammar = (TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Assignment | (TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Declaration ;
class MyAST { public: typedef ASTID ASTID;
typedef std::vector<Token> Source; ASTID ID; std::vector<std::shared_ptr<MyAST>> children; MyAST(ASTID id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end) : ID(id) { } void addChild(const std::shared_ptr<MyAST>& child) { children.push_back(child); } // Recursive function to print the AST tree void printTree(int depth = 0) const { // Print the current node with indentation based on the depth std::string indent(depth * 2, ' '); std::cout << indent << "Node: " << astIDToString(ID) << std::endl; // Recursively print children for (const auto& child : children) { child->printTree(depth + 1); } }
// Assignment, // Declaration // Helper function to convert ASTID enum to string for printing std::string astIDToString(ASTID id) const { switch (id) { case ASTID::Assignment: return "Assignment"; case ASTID::Declaration: return "Declaration"; default: return "Unknown"; } } };
std::vector<std::shared_ptr
> ast; class Error { public: std::vector
::const_iterator START; Error(int id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end) : START(start) { } bool operator < (const Error& err) const { return START < err.START; }
};
int main() { std::vector
errors; std::vector<Token> input; input.push_back(Token{ TokenKind::A, "", 0, 0 }); input.push_back(Token{ TokenKind::Equal, "", 0, 0 }); input.push_back(Token{ TokenKind::B, "", 0, 0 }); input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 }); input.push_back(Token{ TokenKind::A, "", 0, 0 }); input.push_back(Token{ TokenKind::B, "", 0, 0 }); input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 }); parserlib::cfe::parse(input, grammar, ast, errors); // Draw the AST tree in the console std::cout << "AST Tree:" << std::endl; for (const auto& node : ast) { node->printTree(); // Print the AST tree starting from each root node } return 0;
}
I got the build error:
-------------- Build: Debug in test_parser_lib (compiler: GNU GCC Compiler)---------------
[ 50.0%] g++.exe -Wall -fexceptions -g -Iinclude -c main.cpp -o obj\Debug\main.o [100.0%] g++.exe -o bin\Debug\test_parser_lib.exe obj\Debug\main.o main.cpp:42:24: error: no match for 'operator>>' (operand types are 'TokenKind' and 'TokenKind') 42 = (TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Assignment ~~~~ ^~~~~~TokenKind TokenKind
main.cpp:43:24: error: no match for 'operator>>' (operand types are 'TokenKind' and 'TokenKind') 43 (TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Declaration ~~~~ ^~~~~~TokenKind TokenKind Process terminated with status 1 (0 minute(s), 0 second(s)) 2 error(s), 0 warning(s) (0 minute(s), 0 second(s))
So, the error line comes from here:
// try to parse the "A = B;" statement // try to parse the "A B;" statement const auto grammar = (TokenKind::A >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Assignment | (TokenKind::A >> TokenKind::B >> TokenKind::Semicolumn) -> ASTID::Declaration ;
Can you help to solve this issue? It looks like the TokenKind can not be used as a "term"?
But in your example code:
const auto grammar = (TokenKind::A->ASTID::A | TokenKind::B->ASTID::B | TokenKind::C->ASTID::C) ;
This works OK.
Thanks.
— Reply to this email directly, view it on GitHub https://github.com/axilmar/parserlib/issues/16#issuecomment-2418958112, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAESDFBJAJIO3YMTQNHK4JTZ353W5AVCNFSM6AAAAABPYODGFOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDIMJYHE2TQMJRGI . You are receiving this because you commented.Message ID: @.***>
I am preparing a new version of the library, so these issues will be resolved albeit in a different manner.
At the moment, rename your token class to MyToken or use a typedef to distinguish between the two.
On Thu, Oct 17, 2024 at 10:59 AM ollydbg @.***> wrote:
I see another issue:
In the testing code of a standalone parser, there are some code like:
struct Token { TokenKind kind; std::string lexeme; int row; int column;
bool operator == (TokenKind tk) const { return kind == tk; }
};
std::vector
tokens; And in the library's header file, there are same class Token, see here:
namespace parserlib::cfe {
/** * A token. * * It is created from tokenizing an input, and it is fed to a parser to parse a language grammar. * * @param TokenID_ id of token. * * @param Source_ source for the token. */ template <class TokenID_ = int, class Source_ = core::SourceString<>> class Token { public: /** * Token id type. */ typedef TokenID_ TokenID; /** * Source type. */ typedef Source_ Source; /** * Source iterator type. */ typedef typename Source::const_iterator SourceIterator; /** * Source view type. */ typedef std::basic_string_view<typename Source::value_type> StringView; /** * Constructor. * @param id id of token. * @param startPosition start position of the token into the source. * @param endPosition end position of the the token into the source. */ Token(const TokenID& id, const SourceIterator& startPosition, const SourceIterator& endPosition) : m_id(id) , m_startPosition(startPosition) , m_endPosition(endPosition) { } /** * Returns the token id. * @return the token id. */ const TokenID& getID() const { return m_id; } /** * Returns the start position. * @return the start position. */ const SourceIterator& getStartPosition() const { return m_startPosition; } /** * Returns the end position. * @return the end position. */ const SourceIterator& getEndPosition() const { return m_endPosition; } /** * Returns the source that correponds to this token. * @return the source that correponds to this token. */ StringView getSource(size_t maxChars = -1) const { const auto* src = &*m_startPosition; const size_t size = std::min(maxChars, (size_t)(m_endPosition - m_startPosition)); return StringView(src, size); } /** * Checks if this token has the given id. * @param id id to check. * @return true if this token has the given id, false otherwise. */ bool operator == (const TokenID& id) const { return m_id == id; } /** * Checks if this token does not have the given id. * @param id id to check. * @return true if this token does not have the given id, false otherwise. */ bool operator != (const TokenID& id) const { return m_id != id; } private: TokenID m_id; SourceIterator m_startPosition; SourceIterator m_endPosition; };
So the names will get conflict if you use the
using namespace parserlib::cfe;
— Reply to this email directly, view it on GitHub https://github.com/axilmar/parserlib/issues/16#issuecomment-2418833499, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAESDFFF3ODSKHZOKCOU25DZ35U6JAVCNFSM6AAAAABPYODGFOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDIMJYHAZTGNBZHE . You are receiving this because you commented.Message ID: @.***>
Neither is good without a term() call, as I told you in previous answer.
On Thu, Oct 17, 2024 at 1:00 PM ollydbg @.***> wrote:
const auto grammar = *(TokenKind::A >> TokenKind::Equal ->*ASTID::Assignment | TokenKind::C->*ASTID::Declaration) ;
The above code compiles correctly without error.
But the below code has build errors:
const auto grammar = *(TokenKind::A >> TokenKind::Equal >> TokenKind::B ->*ASTID::Assignment | TokenKind::C->*ASTID::Declaration) ;
I'm not sure why.
The former has one >> and the later has two >> operator.
— Reply to this email directly, view it on GitHub https://github.com/axilmar/parserlib/issues/16#issuecomment-2419094956, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAESDFDGNP5QUIASCUQY373Z36DE7AVCNFSM6AAAAABPYODGFOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDIMJZGA4TIOJVGY . You are receiving this because you commented.Message ID: @.***>
Thanks for the reply. Add the term()
function call for the first Token in the grammar solved my problem.
Here is the full source code:
#include "parserlib.hpp"
#include <iostream>
#include <vector>
#include <string>
#include <memory>
#include <cassert>
// using namespace parserlib;
using namespace parserlib::core;
// using namespace parserlib::cfe;
enum class TokenKind {
A,
B,
C,
Equal,
Semicolumn
};
struct Token {
TokenKind kind;
std::string lexeme;
int row;
int column;
bool operator == (TokenKind tk) const {
return kind == tk;
}
};
std::vector<Token> tokens;
enum class ASTID {
Assignment,
Declaration
};
// try to parse the "A = B;" statement
// try to parse the "A B;" statement
const auto grammar
= *(term(TokenKind::A) >> TokenKind::Equal >> TokenKind::B >> TokenKind::Semicolumn ->* ASTID::Assignment
| term(TokenKind::A) >> TokenKind::B >> TokenKind::Semicolumn ->* ASTID::Declaration)
;
class MyAST {
public:
typedef ASTID ASTID;
typedef std::vector<Token> Source;
ASTID ID;
std::vector<std::shared_ptr<MyAST>> children;
MyAST(ASTID id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end)
: ID(id)
{
}
void addChild(const std::shared_ptr<MyAST>& child) {
children.push_back(child);
}
// Recursive function to print the AST tree
void printTree(int depth = 0) const {
// Print the current node with indentation based on the depth
std::string indent(depth * 2, ' ');
std::cout << indent << "Node: " << astIDToString(ID) << std::endl;
// Recursively print children
for (const auto& child : children) {
child->printTree(depth + 1);
}
}
// Assignment,
// Declaration
// Helper function to convert ASTID enum to string for printing
std::string astIDToString(ASTID id) const {
switch (id) {
case ASTID::Assignment: return "Assignment";
case ASTID::Declaration: return "Declaration";
default: return "Unknown";
}
}
};
std::vector<std::shared_ptr<MyAST>> ast;
class Error {
public:
std::vector<Token>::const_iterator START;
Error(int id, std::vector<Token>::const_iterator start, std::vector<Token>::const_iterator end)
: START(start)
{
}
bool operator < (const Error& err) const {
return START < err.START;
}
};
int main()
{
std::vector<Error> errors;
std::vector<Token> input;
input.push_back(Token{ TokenKind::A, "", 0, 0 });
input.push_back(Token{ TokenKind::Equal, "", 0, 0 });
input.push_back(Token{ TokenKind::B, "", 0, 0 });
input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 });
input.push_back(Token{ TokenKind::A, "", 0, 0 });
input.push_back(Token{ TokenKind::B, "", 0, 0 });
input.push_back(Token{ TokenKind::Semicolumn, "", 0, 0 });
parserlib::cfe::parse(input, grammar, ast, errors);
// Draw the AST tree in the console
std::cout << "AST Tree:" << std::endl;
for (const auto& node : ast) {
node->printTree(); // Print the AST tree starting from each root node
}
return 0;
}
And here is the result of the running code:
AST Tree:
Node: Assignment
Node: Declaration
I think I need to find a better way to print the ast tree inside the console window.
I got a lot of error, see below when using GCC compiler:
I just copied the CFE example code to my "main.cpp".
Any ideas?
I guess you are using MSVC compiler, which is more permissive?