nixd/Sema: the semantic analysis for our new parser

default.nix

@@ -3,6 +3,7 @@
 , bison
 , boost182
 , flex
+, fmt
 , gtest
 , libbacktrace
 , lit
@@ -41,6 +42,7 @@ stdenv.mkDerivation {
     gtest
     boost182
     llvmPackages.llvm
+    fmt
   ];
 
   env.CXXFLAGS = "-include ${nix.dev}/include/nix/config.h";

meson.build

@@ -31,6 +31,7 @@ gtest_main = dependency('gtest_main')
 
 llvm = dependency('llvm')
 boost = dependency('boost')
+fmt = dependency('fmt')
 
 cpp = meson.get_compiler('cpp')
 backtrace = cpp.find_library('backtrace')

nixd/include/nixd/Basic/Diagnostic.h

@@ -0,0 +1,156 @@
+/// Diagnostic.h, diagnostic types and definitions
+///
+/// Diagnostics are structures with a main message,
+/// and optionally some additional information (body).
+///
+/// For diagnostics with a body,
+/// they may need a special overrided function to format the message.
+///
+#pragma once
+
+#include "Range.h"
+#include <optional>
+#include <string>
+
+#include <fmt/args.h>
+
+namespace nixd {
+
+class PartialDiagnostic {
+public:
+  virtual const char *format() const {
+    if (Result.empty())
+      Result = fmt::vformat(message(), Args);
+    return Result.c_str();
+  };
+
+  [[nodiscard]] virtual const char *message() const = 0;
+
+  virtual ~PartialDiagnostic() = default;
+
+  template <class T> PartialDiagnostic &operator<<(const T &Var) {
+    Args.push_back(Var);
+    return *this;
+  }
+
+protected:
+  mutable std::string Result;
+  fmt::dynamic_format_arg_store<fmt::format_context> Args;
+  /// Location of this diagnostic
+  RangeIdx Range;
+};
+
+class Note : public PartialDiagnostic {
+public:
+  /// Internal kind
+  enum NoteKind {
+#define DIAG_NOTE(SNAME, CNAME, MESSAGE) NK_##CNAME,
+#include "NoteKinds.inc"
+#undef DIAG_NOTE
+  };
+
+  Note(RangeIdx Range, NoteKind Kind) : Range(Range), Kind(Kind) {}
+
+  template <class T> PartialDiagnostic &operator<<(const T &Var) {
+    Args.push_back(Var);
+    return *this;
+  }
+
+  NoteKind kind() const { return Kind; }
+
+  [[nodiscard]] static const char *message(NoteKind Kind);
+
+  [[nodiscard]] const char *message() const override { return message(kind()); }
+
+  RangeIdx range() const { return Range; }
+
+private:
+  RangeIdx Range;
+  NoteKind Kind;
+};
+
+/// The super class for all diagnostics.
+/// concret diagnostic types are defined in Diagnostic*.inc
+class Diagnostic : public PartialDiagnostic {
+public:
+  /// Each diagnostic contains a severity field,
+  /// should be "Fatal", "Error" or "Warning"
+  /// this will affect the eval process.
+  ///
+  /// "Fatal"   -- shouldn't eval the code, e.g. parsing error.
+  /// "Error"   -- trigger an error in nix, but we can eval the code.
+  /// "Warning" -- just a warning.
+  /// "Note"    -- some additional information about the error.
+  enum Severity { DS_Fatal, DS_Error, DS_Warning };
+
+  /// Internal kind
+  enum DiagnosticKind {
+#define DIAG(SNAME, CNAME, SEVERITY, MESSAGE) DK_##CNAME,
+#include "DiagnosticKinds.inc"
+#undef DIAG
+  };
+
+  Diagnostic(RangeIdx Range, DiagnosticKind Kind) : Range(Range), Kind(Kind) {}
+
+  [[nodiscard]] DiagnosticKind kind() const { return Kind; };
+
+  [[nodiscard]] static Severity severity(DiagnosticKind Kind);
+
+  [[nodiscard]] static const char *message(DiagnosticKind Kind);
+
+  [[nodiscard]] const char *message() const override { return message(kind()); }
+
+  /// Short name, switch name.
+  /// There might be a human readable short name that controls the diagnostic
+  /// For example, one may pass -Wno-dup-formal to suppress duplicated formals.
+  /// A special case for parsing errors, generated from bison
+  /// have the sname "bison"
+  [[nodiscard]] static const char *sname(DiagnosticKind Kind);
+
+  [[nodiscard]] virtual const char *sname() const { return sname(kind()); }
+
+  Note &note(RangeIdx Range, Note::NoteKind Kind) {
+    return *Notes.emplace_back(std::make_unique<Note>(Range, Kind));
+  }
+
+  std::vector<std::unique_ptr<Note>> &notes() { return Notes; }
+
+  RangeIdx range() const { return Range; }
+
+private:
+  /// Location of this diagnostic
+  RangeIdx Range;
+
+  std::vector<std::unique_ptr<Note>> Notes;
+
+  DiagnosticKind Kind;
+};
+
+class DiagnosticEngine {
+public:
+  Diagnostic &diag(RangeIdx Range, Diagnostic::DiagnosticKind Kind) {
+    auto Diag = std::make_unique<Diagnostic>(Range, Kind);
+    switch (getServerity(Diag->kind())) {
+    case Diagnostic::DS_Fatal:
+      return *Fatals.emplace_back(std::move(Diag));
+    case Diagnostic::DS_Error:
+      return *Errors.emplace_back(std::move(Diag));
+    case Diagnostic::DS_Warning:
+      return *Warnings.emplace_back(std::move(Diag));
+    }
+  }
+
+  std::vector<std::unique_ptr<Diagnostic>> &warnings() { return Warnings; }
+  std::vector<std::unique_ptr<Diagnostic>> &errors() { return Errors; }
+  std::vector<std::unique_ptr<Diagnostic>> &fatals() { return Fatals; }
+
+private:
+  Diagnostic::Severity getServerity(Diagnostic::DiagnosticKind Kind) {
+    return Diagnostic::severity(Kind);
+  }
+  std::vector<std::unique_ptr<Diagnostic>> Warnings;
+  std::vector<std::unique_ptr<Diagnostic>> Errors;
+  std::vector<std::unique_ptr<Diagnostic>> Fatals;
+};
+
+} // namespace nixd

nixd/include/nixd/Basic/DiagnosticKinds.inc

@@ -0,0 +1,30 @@
+/// DiagnosticKinds.inc, diagnostic declarations
+
+#ifdef DIAG
+DIAG("let-dynamic", LetDynamic, Error,
+     "dynamic attributes are not allowed in let ... in ... expression")
+DIAG("inherit-dynamic", InheritDynamic, Error,
+     "dynamic attributes are not allowed in inherit")
+DIAG("empty-inherit", EmptyInherit, Warning, "empty inherit expression")
+DIAG("or-identifier", OrIdentifier, Warning,
+     "keyword `or` used as an identifier")
+DIAG("deprecated-url-literal", DeprecatedURL, Warning,
+     "URL literal is deprecated")
+DIAG("deprecated-let", DeprecatedLet, Warning,
+     "using deprecated `let' syntactic sugar `let {{..., body = ...}}' -> "
+     "(rec {{..., body = ...}}).body'")
+DIAG("path-trailing-slash", PathTrailingSlash, Fatal,
+     "path has a trailing slash")
+DIAG("dup-formal", DuplicatedFormal, Error,
+     "duplicated function formal declaration")
+DIAG("dup-formal-arg", DuplicatedFormalToArg, Error,
+     "function argument duplicated to a function formal")
+DIAG("merge-diff-rec", MergeDiffRec, Warning,
+     "merging two attributes with different `rec` modifiers, the latter "
+     "will be implicitly ignored")
+DIAG("bison", BisonParse, Fatal, "{}")
+DIAG("invalid-float", InvalidFloat, Fatal, "invalid float {}")
+DIAG("invalid-integer", InvalidInteger, Fatal, "invalid integer {}")
+DIAG("dup-formal-arg", DuplicatedAttr, Error, "duplicated attr `{}`")
+
+#endif // DIAG

nixd/include/nixd/Basic/NoteKinds.inc

@@ -0,0 +1,10 @@
+/// DiagnosticNodes.inc, note declarations
+
+#ifdef DIAG_NOTE
+DIAG_NOTE("note-prev", PrevDeclared, "previously declared here")
+DIAG_NOTE("merge-diff-rec-this-rec", ThisRecursive,
+          "this attribute set is {}recursive")
+DIAG_NOTE("merge-diff-rec-consider", RecConsider,
+          "while this attribute set is marked as {}recursive, it "
+          "will be considered as {}recursive")
+#endif // DIAG_NOTE

nixd/include/nixd/Sema/EvalContext.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "nixd/Support/GCPool.h"
+
+#include <nix/nixexpr.hh>
+
+namespace nixd {
+
+struct EvalContext {
+  using ES = std::vector<std::pair<nix::PosIdx, nix::Expr *>>;
+  GCPool<nix::Expr> Pool;
+  GCPool<nix::Formals> FormalsPool;
+  GCPool<ES> ESPool;
+};
+
+} // namespace nixd

nixd/include/nixd/Sema/Lowering.h

@@ -0,0 +1,131 @@
+#pragma once
+
+#include "nixd/Basic/Diagnostic.h"
+#include "nixd/Sema/EvalContext.h"
+#include "nixd/Syntax/Nodes.h"
+
+#include <nix/input-accessor.hh>
+#include <nix/nixexpr.hh>
+#include <nix/symbol-table.hh>
+
+namespace nixd {
+
+struct Lowering;
+
+class ExprAttrsBuilder {
+  // The official parser does this work during the lowering process, via
+  // @addAttr in src/libexpr/parser.y
+
+  Lowering &LW;
+  nix::ExprAttrs *Result;
+
+  RangeIdx Range;
+
+  bool Recursive;
+
+  /// let ... in ...
+  /// It is not allowed to use dynamic binds here, so we want to give diagnostic
+  /// to each occurrence.
+  bool IsLet;
+
+  /// Nested attributes, we create a new builder for them, and collapse the map
+  /// while finishing
+  std::map<nix::Symbol, std::unique_ptr<ExprAttrsBuilder>> Nested;
+
+  std::map<syntax::Node *, std::unique_ptr<ExprAttrsBuilder>> DynamicNested;
+
+  // Use a map to detect duplicated fields
+  // Not using the map inside nix::ExprAttrs because we want to preserve the
+  // range
+  std::map<nix::Symbol, const syntax::Node *> Fields;
+
+public:
+  ExprAttrsBuilder(Lowering &LW, RangeIdx Range, bool Recursive, bool IsLet);
+
+  void addAttr(const syntax::Node *Attr, const syntax::Node *Body,
+               bool Recursive);
+  void addAttribute(const syntax::Attribute &A);
+  void addBinds(const syntax::Binds &Binds);
+  void addAttrSet(const syntax::AttrSet &AS);
+  void addInherited(const syntax::InheritedAttribute &IA);
+  nix::ExprAttrs *finish();
+};
+
+struct Lowering {
+  nix::SymbolTable &STable;
+  nix::PosTable &PTable;
+  DiagnosticEngine &Diags;
+  EvalContext &Ctx;
+  const nix::SourcePath &BasePath;
+
+  nix::Expr *lower(const syntax::Node *Root);
+  nix::ExprLambda *lowerFunction(const syntax::Function *Fn);
+  nix::Formal lowerFormal(const syntax::Formal &Formal);
+  nix::AttrPath lowerAttrPath(const syntax::AttrPath &Path);
+  nix::Expr *lowerOp(const syntax::Node *Op);
+
+private:
+  constexpr static std::string_view LessThan = "__lessThan";
+  constexpr static std::string_view Sub = "__sub";
+  constexpr static std::string_view Mul = "__mul";
+  constexpr static std::string_view Div = "__div";
+  constexpr static std::string_view CurPos = "__curPos";
+  constexpr static std::string_view FindFile = "__findFile";
+  constexpr static std::string_view NixPath = "__nixPath";
+
+  nix::Expr *stripIndentation(const syntax::IndStringParts &ISP);
+
+  nix::ExprVar *mkVar(std::string_view Sym) {
+    return mkVar(STable.create(Sym));
+  }
+
+  nix::ExprVar *mkVar(nix::Symbol Sym) {
+    auto *Var = new nix::ExprVar(Sym);
+    Ctx.Pool.record(Var);
+    return Var;
+  };
+
+  nix::ExprOpNot *mkNot(nix::Expr *Body) {
+    auto *OpNot = new nix::ExprOpNot(Body);
+    Ctx.Pool.record(OpNot);
+    return OpNot;
+  }
+
+  nix::ExprCall *mkCall(std::string_view FnName, nix::PosIdx P,
+                        std::vector<nix::Expr *> Args) {
+    nix::ExprVar *Fn = mkVar(FnName);
+    auto *Nix = new nix::ExprCall(P, Fn, std::move(Args));
+    return Ctx.Pool.record(Nix);
+  }
+
+  template <class SyntaxTy>
+  nix::Expr *lowerCallOp(std::string_view FnName, const syntax::Node *Op,
+                         bool SwapOperands = false, bool Not = false) {
+    auto *SOP = dynamic_cast<const SyntaxTy *>(Op);
+    assert(SOP && "types are not matching between op pointer & syntax type!");
+    nix::Expr *LHS = lower(SOP->LHS);
+    nix::Expr *RHS = lower(SOP->RHS);
+    if (SwapOperands)
+      std::swap(LHS, RHS);
+    nix::PosIdx P = SOP->OpRange.Begin;
+    nix::ExprCall *Call = mkCall(FnName, P, {LHS, RHS});
+    if (Not)
+      return mkNot(Call);
+    return Call;
+  }
+
+  /// The bin-op expression is actually legal in the nix evaluator
+  /// OpImpl -> nix::ExprOpImpl
+  template <class NixTy, class SyntaxTy>
+  NixTy *lowerLegalOp(const syntax::Node *Op) {
+    auto *SOP = dynamic_cast<const SyntaxTy *>(Op);
+    assert(SOP && "types are not matching between op pointer & syntax type!");
+    nix::Expr *LHS = lower(SOP->LHS);
+    nix::Expr *RHS = lower(SOP->RHS);
+    nix::PosIdx P = SOP->OpRange.Begin;
+    auto *Nix = new NixTy(P, LHS, RHS);
+    return Ctx.Pool.record(Nix);
+  }
+};
+
+} // namespace nixd

nixd/include/nixd/Support/Position.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "nixd/Basic/Range.h"
 #include "nixd/Nix/PosAdapter.h"
-#include "nixd/Syntax/Range.h"
 
 #include "lspserver/Protocol.h"