From 1bafc1a6a85ab334e8141d6bc66bcbdcc9eb7552 Mon Sep 17 00:00:00 2001 From: Lauro Oyen Date: Sat, 7 Dec 2024 13:27:24 +0100 Subject: [PATCH 1/4] Add binding generator target with initial entry point code --- tools/CMakeLists.txt | 7 + .../binding-generator-main.cpp | 161 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 tools/slang-binding-generator/binding-generator-main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index dfa19d42ff..8e1a80ac9a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -52,6 +52,13 @@ generator( compiler-core slang-cpp-parser ) +generator( + slang-binding-generator + USE_FEWER_WARNINGS + LINK_WITH_PRIVATE + compiler-core + slang-cpp-parser +) generator(slang-embed) generator(slang-generate USE_FEWER_WARNINGS) generator(slang-lookup-generator LINK_WITH_PRIVATE compiler-core) diff --git a/tools/slang-binding-generator/binding-generator-main.cpp b/tools/slang-binding-generator/binding-generator-main.cpp new file mode 100644 index 0000000000..e31f18a398 --- /dev/null +++ b/tools/slang-binding-generator/binding-generator-main.cpp @@ -0,0 +1,161 @@ +#include "compiler-core/slang-diagnostic-sink.h" +#include "compiler-core/slang-lexer.h" +#include "compiler-core/slang-name.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-file-system.h" +#include "core/slang-io.h" +#include "core/slang-list.h" +#include "core/slang-string-slice-pool.h" +#include "core/slang-string-util.h" +#include "core/slang-string.h" +#include "core/slang-writer.h" +#include "slang-com-helper.h" +#include "slang-cpp-parser/diagnostics.h" +#include "slang-cpp-parser/file-util.h" +#include "slang-cpp-parser/node-tree.h" +#include "slang-cpp-parser/node.h" +#include "slang-cpp-parser/options.h" +#include "slang-cpp-parser/parser.h" +#include "slang-cpp-parser/unit-test.h" + +#include +#include +#include + +namespace BindingGenerator +{ + +using namespace Slang; +using namespace CppParse; + +class App +{ +public: + SlangResult execute(const Options& options); + + SlangResult executeWithArgs(int argc, const char* const* argv); + + const Options& getOptions() const { return m_options; } + + App(DiagnosticSink* sink, SourceManager* sourceManager, RootNamePool* rootNamePool) + : m_sink(sink), m_sourceManager(sourceManager), m_slicePool(StringSlicePool::Style::Default) + { + m_namePool.setRootNamePool(rootNamePool); + } + +protected: + NamePool m_namePool; + + Options m_options; + DiagnosticSink* m_sink; + SourceManager* m_sourceManager; + + StringSlicePool m_slicePool; +}; + +SlangResult App::execute(const Options& options) +{ + m_options = options; + + if (options.m_runUnitTests) + { + SLANG_RETURN_ON_FAIL(UnitTestUtil::run()); + } + + IdentifierLookup identifierLookup; + identifierLookup.initDefault(options.m_markPrefix.getUnownedSlice()); + + NodeTree tree(&m_slicePool, &m_namePool, &identifierLookup); + + // Read in each of the input files + for (Index i = 0; i < m_options.m_inputPaths.getCount(); ++i) + { + String inputPath; + + if (m_options.m_inputDirectory.getLength()) + { + inputPath = Path::combine(m_options.m_inputDirectory, m_options.m_inputPaths[i]); + } + else + { + inputPath = m_options.m_inputPaths[i]; + } + + // Read the input file + String contents; + SLANG_RETURN_ON_FAIL(FileUtil::readAllText(inputPath, m_sink, contents)); + + PathInfo pathInfo = PathInfo::makeFromString(inputPath); + + SourceFile* sourceFile = m_sourceManager->createSourceFileWithString(pathInfo, contents); + + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, options); + + Parser parser(&tree, m_sink); + SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &m_options)); + } + + SLANG_RETURN_ON_FAIL(tree.calcDerivedTypes(m_sink)); + + // Dump out the tree + if (options.m_dump) + { + { + StringBuilder buf; + tree.getRootNode()->dump(0, buf); + m_sink->writer->write(buf.getBuffer(), buf.getLength()); + } + } + + return SLANG_OK; +} + +SlangResult App::executeWithArgs(int argc, const char* const* argv) +{ + Options options; + OptionsParser optionsParser; + SLANG_RETURN_ON_FAIL(optionsParser.parse(argc, argv, m_sink, options)); + SLANG_RETURN_ON_FAIL(execute(options)); + return SLANG_OK; +} + +} // namespace BindingGenerator + +int main(int argc, const char* const* argv) +{ + using namespace Slang; + using namespace BindingGenerator; + + ComPtr writer(new FileWriter(stderr, WriterFlag::AutoFlush)); + + RootNamePool rootNamePool; + + SourceManager sourceManager; + sourceManager.initialize(nullptr, nullptr); + + DiagnosticSink sink(&sourceManager, Lexer::sourceLocationLexer); + sink.writer = writer; + + App app(&sink, &sourceManager, &rootNamePool); + + try + { + if (SLANG_FAILED(app.executeWithArgs(argc - 1, argv + 1))) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); + return 1; + } + if (sink.getErrorCount()) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::extractorFailed); + return 1; + } + } + catch (...) + { + sink.diagnose(SourceLoc(), CPPDiagnostics::internalError); + return 1; + } + + return 0; +} From 102cf24171c72e06a75c1fab56397ec7e4920090 Mon Sep 17 00:00:00 2001 From: Lauro Oyen Date: Sun, 8 Dec 2024 21:18:44 +0100 Subject: [PATCH 2/4] Always parse all node kinds --- include/slang.h | 2 - tools/slang-cpp-parser/parser.cpp | 105 +++++++++++++-------------- tools/slang-cpp-parser/parser.h | 12 --- tools/slang-cpp-parser/unit-test.cpp | 10 --- 4 files changed, 51 insertions(+), 78 deletions(-) diff --git a/include/slang.h b/include/slang.h index 2dfee6b281..9959bfa5c6 100644 --- a/include/slang.h +++ b/include/slang.h @@ -1729,8 +1729,6 @@ public: \ typedef slang::IGlobalSession SlangSession; - typedef struct SlangProgramLayout SlangProgramLayout; - /*! @brief A request for one or more compilation actions to be performed. */ diff --git a/tools/slang-cpp-parser/parser.cpp b/tools/slang-cpp-parser/parser.cpp index 1edc38ac73..9fcf6a6b98 100644 --- a/tools/slang-cpp-parser/parser.cpp +++ b/tools/slang-cpp-parser/parser.cpp @@ -16,44 +16,8 @@ SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t)); // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) - : m_sink(sink), m_nodeTree(nodeTree), m_nodeTypeEnabled(0) + : m_sink(sink), m_nodeTree(nodeTree) { - // Enable types by default - const Node::Kind defaultEnabled[] = { - Node::Kind::ClassType, - Node::Kind::StructType, - Node::Kind::Namespace, - Node::Kind::AnonymousNamespace, - Node::Kind::Field, - - // These are disabled by default because AST uses macro magic to build up the types - // Node::Type::TypeDef, - // Node::Type::Enum, - // Node::Type::EnumClass, - - Node::Kind::Callable, - }; - setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); -} - -void Parser::setKindEnabled(Node::Kind kind, bool isEnabled) -{ - if (isEnabled) - { - m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind)); - } - else - { - m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind)); - } -} - -void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled) -{ - for (Index i = 0; i < kindsCount; ++i) - { - setKindEnabled(kinds[i], isEnabled); - } } bool Parser::_isMarker(const UnownedStringSlice& name) @@ -413,6 +377,33 @@ SlangResult Parser::_parseEnum() break; } + // TODO: + // The SlangImageFormat enum #includes "slang-image-format-defs.h" to define its cases. + // Since this parser has no preprocessor, it can't unfold those enum cases. For now skip + // the # directives in the enum body, resulting in an empty parsed enum. Maybe fix it by + // unfolding the enum manually where it is #included, resulting in some code duplication. + if (tokenType == TokenType::Pound) + { + while (m_reader.peekTokenType() == TokenType::Pound) + { + Token token = m_reader.peekToken(); + if (token.flags & TokenFlag::AtStartOfLine) + { + m_reader.advanceToken(); + for (;;) + { + auto t = m_reader.peekToken(); + if (t.type == TokenType::EndOfFile || (t.flags & TokenFlag::AtStartOfLine)) + { + break; + } + m_reader.advanceToken(); + } + } + } + break; + } + RefPtr caseNode(new EnumCaseNode); // We could also check if the name is a valid identifier for name, for now just assume. @@ -1013,6 +1004,13 @@ SlangResult Parser::_maybeParseType( // TODO(JS): // Doesn't handle all the modifiers just (*SomeName) + // Skip the class name in case of a pointer to member function (Class::*Member) + if (m_reader.peekTokenType() == TokenType::Identifier) + { + m_reader.advanceToken(); + expect(TokenType::Scope); + } + SLANG_RETURN_ON_FAIL(expect(TokenType::OpMul)); outNameCursor = m_reader.getCursor(); SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); @@ -1471,6 +1469,9 @@ SlangResult Parser::_parseTypeDef() // Consume the typedef SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + _maybeConsume(IdentifierStyle::Struct); + _maybeConsume(IdentifierStyle::Enum); + Token nameToken; // Parse the type List toks; @@ -1484,6 +1485,17 @@ SlangResult Parser::_parseTypeDef() if (Node::lookupNameInScope(m_currentScope, nameToken.getContent())) { + // TODO: + // In slang.h these types are defined twice as different sizes and then conditionally + // enabled based on a define. Since this parser has no preprocessor, it can't handle this. + // Maybe fix it by moving all of the platform-detection stuff out to a separate file as + // suggested here: https://github.com/shader-slang/slang/pull/954#discussion_r278687475 + if (nameToken.getContent() == "SlangInt" || nameToken.getContent() == "SlangUInt" || + nameToken.getContent() == "SlangSSizeT" || nameToken.getContent() == "SlangSizeT") + { + return SLANG_OK; + } + m_sink->diagnose( nameToken.loc, CPPDiagnostics::identifierAlreadyDefined, @@ -2129,15 +2141,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) } case IdentifierStyle::TypeDef: { - if (isTypeEnabled(Node::Kind::TypeDef)) - { - SLANG_RETURN_ON_FAIL(_parseTypeDef()); - } - else - { - m_reader.advanceToken(); - SLANG_RETURN_ON_FAIL(_consumeToSync()); - } + SLANG_RETURN_ON_FAIL(_parseTypeDef()); break; } default: @@ -2149,14 +2153,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) Node::Kind kind = _toNodeKind(style); SLANG_ASSERT(kind != Node::Kind::Invalid); - if (isTypeEnabled(kind)) - { - SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); - } - else - { - SLANG_RETURN_ON_FAIL(_maybeConsumeScope()); - } + SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); } else { diff --git a/tools/slang-cpp-parser/parser.h b/tools/slang-cpp-parser/parser.h index 519ad69355..c3ccf9656a 100644 --- a/tools/slang-cpp-parser/parser.h +++ b/tools/slang-cpp-parser/parser.h @@ -13,8 +13,6 @@ using namespace Slang; class Parser { public: - typedef uint32_t NodeTypeBitType; - SlangResult expect(TokenType type, Token* outToken = nullptr); bool advanceIfMarker(Token* outToken = nullptr); @@ -29,14 +27,6 @@ class Parser /// Parse the contents of the source file SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); - void setKindEnabled(Node::Kind kind, bool isEnabled = true); - bool isTypeEnabled(Node::Kind kind) - { - return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; - } - - void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); - Parser(NodeTree* nodeTree, DiagnosticSink* sink); protected: @@ -92,8 +82,6 @@ class Parser /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( SlangResult _consumeBalancedParens(); - NodeTypeBitType m_nodeTypeEnabled; - TokenList m_tokenList; TokenReader m_reader; diff --git a/tools/slang-cpp-parser/unit-test.cpp b/tools/slang-cpp-parser/unit-test.cpp index 70851fd7c8..7320e708e8 100644 --- a/tools/slang-cpp-parser/unit-test.cpp +++ b/tools/slang-cpp-parser/unit-test.cpp @@ -91,16 +91,6 @@ static const char someSource[] = "class ISomeInterface\n" Parser parser(&tree, &state.m_sink); - - { - const Node::Kind enableKinds[] = { - Node::Kind::Enum, - Node::Kind::EnumClass, - Node::Kind::EnumCase, - Node::Kind::TypeDef}; - parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); - } - SlangResult res = parser.parse(sourceOrigin, &state.m_options); if (state.m_sink.outputBuffer.getLength()) From d778e6f55fcf403deb75b921389fd233dbad784a Mon Sep 17 00:00:00 2001 From: Lauro Oyen Date: Mon, 9 Dec 2024 18:09:37 +0100 Subject: [PATCH 3/4] Parse GUIDs in COM_INTERFACE macros --- source/core/slang-string-util.cpp | 4 +- tools/slang-cpp-parser/node.cpp | 20 ++++++++++ tools/slang-cpp-parser/node.h | 4 +- tools/slang-cpp-parser/parser.cpp | 56 ++++++++++++++++++++-------- tools/slang-cpp-parser/parser.h | 1 + tools/slang-cpp-parser/unit-test.cpp | 2 + 6 files changed, 70 insertions(+), 17 deletions(-) diff --git a/source/core/slang-string-util.cpp b/source/core/slang-string-util.cpp index 6f1dc2ccb8..3da4084618 100644 --- a/source/core/slang-string-util.cpp +++ b/source/core/slang-string-util.cpp @@ -690,16 +690,18 @@ String StringUtil::replaceAll( } int radix = 10; + auto isDigit = CharUtil::isDigit; auto getDigit = CharUtil::getDecimalDigitValue; if (cur + 1 < end && *cur == '0' && (*(cur + 1) == 'x' || *(cur + 1) == 'X')) { radix = 16; + isDigit = CharUtil::isHexDigit; getDigit = CharUtil::getHexDigitValue; cur += 2; } // We need at least one digit - if (cur >= end || !CharUtil::isDigit(*cur)) + if (cur >= end || !isDigit(*cur)) { return SLANG_FAIL; } diff --git a/tools/slang-cpp-parser/node.cpp b/tools/slang-cpp-parser/node.cpp index 16484ead39..af15508580 100644 --- a/tools/slang-cpp-parser/node.cpp +++ b/tools/slang-cpp-parser/node.cpp @@ -688,6 +688,26 @@ void ClassLikeNode::dump(int indentCount, StringBuilder& out) out << " {\n"; + if (m_guid != Guid()) + { + _indent(indentCount + 1, out); + StringUtil::appendFormat( + out, + "COM_INTERFACE(0x%08lx, 0x%04hx, 0x%04hx, {0x%02hhx, 0x%02hhx, 0x%02hhx, 0x%02hhx, " + "0x%02hhx, 0x%02hhx, 0x%02hhx, 0x%02hhx})\n", + m_guid.data1, + m_guid.data2, + m_guid.data3, + m_guid.data4[0], + m_guid.data4[1], + m_guid.data4[2], + m_guid.data4[3], + m_guid.data4[4], + m_guid.data4[5], + m_guid.data4[6], + m_guid.data4[7]); + } + for (Node* child : m_children) { child->dump(indentCount + 1, out); diff --git a/tools/slang-cpp-parser/node.h b/tools/slang-cpp-parser/node.h index 4dad9473e0..04cee8d715 100644 --- a/tools/slang-cpp-parser/node.h +++ b/tools/slang-cpp-parser/node.h @@ -316,7 +316,7 @@ struct ClassLikeNode : public ScopeNode virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; ClassLikeNode(Kind kind) - : Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr) + : Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr), m_guid(Guid()) { SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType); } @@ -333,6 +333,8 @@ struct ClassLikeNode : public ScopeNode Token m_super; ///< Super class name ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or ///< nullptr if base) + + Guid m_guid; ///< The guid associated with this type }; struct CallableNode : public Node diff --git a/tools/slang-cpp-parser/parser.cpp b/tools/slang-cpp-parser/parser.cpp index 9fcf6a6b98..32f68216b3 100644 --- a/tools/slang-cpp-parser/parser.cpp +++ b/tools/slang-cpp-parser/parser.cpp @@ -1109,32 +1109,58 @@ SlangResult Parser::_parseSpecialMacro() Token name; SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - List params; + const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); - if (m_reader.peekTokenType() == TokenType::LParent) + if (suffix == "COM_INTERFACE") { - // Mark the start - auto startCursor = m_reader.getCursor(); + return _parseGuid(); + } - // Consume the params + if (m_reader.peekTokenType() == TokenType::LParent) + { SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + } - auto endCursor = m_reader.getCursor(); - m_reader.setCursor(startCursor); + return SLANG_OK; +} - while (!m_reader.isAtCursor(endCursor)) +SlangResult Parser::_parseGuid() +{ + Guid guid{}; + Token guidToken; + Int value; + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::IntegerLiteral, &guidToken)); + StringUtil::parseInt(guidToken.getContent(), value); + guid.data1 = value; + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + SLANG_RETURN_ON_FAIL(expect(TokenType::IntegerLiteral, &guidToken)); + StringUtil::parseInt(guidToken.getContent(), value); + guid.data2 = value; + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + SLANG_RETURN_ON_FAIL(expect(TokenType::IntegerLiteral, &guidToken)); + StringUtil::parseInt(guidToken.getContent(), value); + guid.data3 = value; + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + SLANG_RETURN_ON_FAIL(expect(TokenType::LBrace)); + for (Index i = 0; i < 8; ++i) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::IntegerLiteral, &guidToken)); + StringUtil::parseInt(guidToken.getContent(), value); + guid.data4[i] = value; + if (i < 7) { - params.add(m_reader.advanceToken()); + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); } } + SLANG_RETURN_ON_FAIL(expect(TokenType::RBrace)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - // Can do special handling here - const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); + ClassLikeNode* node = as(m_currentScope); - if (suffix == "COM_INTERFACE") - { - // TODO(JS): It's a com interface. Extact the GUID - } + node->m_guid = guid; return SLANG_OK; } diff --git a/tools/slang-cpp-parser/parser.h b/tools/slang-cpp-parser/parser.h index c3ccf9656a..675fdccde1 100644 --- a/tools/slang-cpp-parser/parser.h +++ b/tools/slang-cpp-parser/parser.h @@ -44,6 +44,7 @@ class Parser SlangResult _parseTypeDef(); SlangResult _parseEnum(); + SlangResult _parseGuid(); SlangResult _parseMarker(); SlangResult _parseSpecialMacro(); diff --git a/tools/slang-cpp-parser/unit-test.cpp b/tools/slang-cpp-parser/unit-test.cpp index 7320e708e8..da8542be45 100644 --- a/tools/slang-cpp-parser/unit-test.cpp +++ b/tools/slang-cpp-parser/unit-test.cpp @@ -42,6 +42,8 @@ struct TestState static const char someSource[] = "class ISomeInterface\n" "{\n" + " SLANG_COM_INTERFACE(0x514027d8, 0x23d1, 0x4093, " + "{0x94,0x85,0xb9,0x2c,0x06,0x95,0x7f,0x5e})\n" " public:\n" " virtual int SLANG_MCALL someMethod(int a, int b) const = 0;\n" " virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" From 674153bcacedc436684b914aa1b9432c6ea9f663 Mon Sep 17 00:00:00 2001 From: Lauro Oyen Date: Mon, 9 Dec 2024 20:46:23 +0100 Subject: [PATCH 4/4] Run binding generator as part of build process --- source/slang/CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/slang/CMakeLists.txt b/source/slang/CMakeLists.txt index 2674988969..3e7c0f5234 100644 --- a/source/slang/CMakeLists.txt +++ b/source/slang/CMakeLists.txt @@ -101,6 +101,18 @@ target_include_directories( INTERFACE ${SLANG_REFLECT_OUTPUT_DIR} ) +# +# generate language bindings +# + +add_custom_target( + generate_language_bindings + COMMAND + slang-binding-generator -d "${slang_SOURCE_DIR}/include" slang.h -unit-test + DEPENDS "${slang_SOURCE_DIR}/include/slang.h" slang-binding-generator + VERBATIM +) + # # generated lookup tables # @@ -212,6 +224,7 @@ set(slang_build_args ${SLANG_RECORD_REPLAY_SYSTEM} REQUIRES copy_slang_headers + generate_language_bindings ) set(slang_link_args LINK_WITH_PRIVATE