irc: improve multi-line strings; complete list concat and dynamic attrs

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I64e53c68d90b62f3ca306865ceda32af6a6a6964

src/irc/evaluator.cpp

@@ -130,6 +130,14 @@ struct Evaluator::Impl {
       // We can use EvalState's searchPath to resolve
       auto path = state.findFile(n->value);
       v.mkPath(path);
+    } else if (auto* n = node->get_if<ListNode>()) {
+      // Evaluate list - allocate and populate
+      auto builder = state.buildList(n->elements.size());
+      for (size_t i = 0; i < n->elements.size(); i++) {
+        builder.elems[i] = state.allocValue();
+        eval_node(n->elements[i], *builder.elems[i], env);
+      }
+      v.mkList(builder);
     } else if (auto* n = node->get_if<VarNode>()) {
       Value* bound = env ? env->lookup(n->index) : nullptr;
       if (!bound && env && n->name.has_value()) {
@@ -316,15 +324,34 @@ struct Evaluator::Impl {
             state.error<EvalError>("type error in comparison").debugThrow();
           }
           break;
-        case BinaryOp::CONCAT:
-          // TODO: ++ list concatenation requires accessing private Nix Value payload
-          // For now, delegate to Nix's concatLists or implement via builtins
-          // Parser recognizes ++ but evaluator not yet fully implemented
-          state
-              .error<EvalError>(
-                  "list concatenation (++) not yet fully implemented - use builtins.concatLists")
-              .debugThrow();
+        case BinaryOp::CONCAT: {
+          // List concatenation: left ++ right
+          if (left->type() != nList || right->type() != nList) {
+            state.error<EvalError>("list concatenation requires two lists").debugThrow();
+          }
+
+          size_t left_size = left->listSize();
+          size_t right_size = right->listSize();
+          size_t total_size = left_size + right_size;
+
+          auto builder = state.buildList(total_size);
+          auto left_view = left->listView();
+          auto right_view = right->listView();
+
+          // Copy elements from left list
+          size_t idx = 0;
+          for (auto elem : left_view) {
+            builder.elems[idx++] = elem;
+          }
+
+          // Copy elements from right list
+          for (auto elem : right_view) {
+            builder.elems[idx++] = elem;
+          }
+
+          v.mkList(builder);
           break;
+        }
         case BinaryOp::MERGE: {
           // // is attrset merge - right overrides left
           if (left->type() != nAttrs || right->type() != nAttrs) {
@@ -418,16 +445,33 @@ struct Evaluator::Impl {
         // For recursive attrsets, create environment where all bindings can
         // see each other
         attr_env = make_env(env);
-        for (const auto& [key, val] : n->attrs) {
-          Value* thunk = make_thunk(val, attr_env);
-          attr_env->bind(thunk);
+        for (const auto& binding : n->attrs) {
+          if (!binding.is_dynamic()) {
+            Value* thunk = make_thunk(binding.value, attr_env);
+            attr_env->bind(thunk);
+          }
         }
       }
 
       // Attributes should be lazy, so store as thunks and not evaluated values
-      for (const auto& [key, val] : n->attrs) {
-        Value* attr_val = make_thunk(val, attr_env);
-        bindings.insert(state.symbols.create(key), attr_val);
+      for (const auto& binding : n->attrs) {
+        Value* attr_val = make_thunk(binding.value, attr_env);
+
+        if (binding.is_dynamic()) {
+          // Evaluate key expression to get attribute name
+          Value* key_val = state.allocValue();
+          eval_node(binding.dynamic_name, *key_val, attr_env);
+          force(key_val);
+
+          if (key_val->type() != nString) {
+            state.error<EvalError>("dynamic attribute name must evaluate to a string").debugThrow();
+          }
+
+          std::string key_str = std::string(key_val->c_str());
+          bindings.insert(state.symbols.create(key_str), attr_val);
+        } else {
+          bindings.insert(state.symbols.create(binding.static_name.value()), attr_val);
+        }
       }
 
       v.mkAttrs(bindings.finish());

src/irc/ir_gen.cpp

@@ -122,11 +122,17 @@ struct IRGenerator::Impl {
     if (auto* n = node.get_if<AttrsetNode>()) {
       AttrsetNode attrs(n->recursive, n->line);
       name_resolver.enter_scope();
-      for (const auto& [key, val] : n->attrs) {
-        name_resolver.bind(key);
+      for (const auto& binding : n->attrs) {
+        if (!binding.is_dynamic()) {
+          name_resolver.bind(binding.static_name.value());
+        }
       }
-      for (const auto& [key, val] : n->attrs) {
-        attrs.attrs.push_back({key, convert(val)});
+      for (const auto& binding : n->attrs) {
+        if (binding.is_dynamic()) {
+          attrs.attrs.push_back(AttrBinding(convert(binding.dynamic_name), convert(binding.value)));
+        } else {
+          attrs.attrs.push_back(AttrBinding(binding.static_name.value(), convert(binding.value)));
+        }
       }
       name_resolver.exit_scope();
       return std::make_shared<Node>(attrs);

src/irc/parser.cpp

@@ -426,7 +426,30 @@ private:
           pos += 3;
           continue;
         } else if (pos + 2 < input.size() && input[pos + 2] == '\\') {
-          // ''\ -> escape for backslash
+          // ''\ -> check what follows
+          if (pos + 3 < input.size()) {
+            char next = input[pos + 3];
+            if (next == 'n') {
+              raw_content += '\n';
+              pos += 4;
+              continue;
+            } else if (next == 'r') {
+              raw_content += '\r';
+              pos += 4;
+              continue;
+            } else if (next == 't') {
+              raw_content += '\t';
+              pos += 4;
+              continue;
+            } else if (next == ' ' || next == '\t') {
+              // ''\ before whitespace - preserve the whitespace (mark it specially)
+              raw_content += "\x01"; // Use control char as marker for preserved whitespace
+              raw_content += next;
+              pos += 4;
+              continue;
+            }
+          }
+          // Default: literal backslash
           raw_content += '\\';
           pos += 3;
           continue;
@@ -486,12 +509,17 @@ private:
     }
 
     // Find minimum indentation (spaces/tabs at start of non-empty lines)
+    // \x01 marker indicates preserved whitespace (from ''\ escape)
     size_t min_indent = std::string::npos;
     for (const auto& line : lines) {
       if (line.empty())
         continue; // Skip empty lines when calculating indentation
       size_t indent = 0;
-      for (char c : line) {
+      for (size_t i = 0; i < line.size(); i++) {
+        char c = line[i];
+        // If we hit the preserved whitespace marker, stop counting indentation
+        if (c == '\x01')
+          break;
         if (c == ' ' || c == '\t')
           indent++;
         else
@@ -504,7 +532,7 @@ private:
     if (min_indent == std::string::npos)
       min_indent = 0;
 
-    // Strip min_indent from all lines
+    // Strip min_indent from all lines and remove \x01 markers
     std::string result;
     for (size_t i = 0; i < lines.size(); i++) {
       const auto& line = lines[i];
@@ -513,9 +541,25 @@ private:
         if (i + 1 < lines.size())
           result += '\n';
       } else {
-        // Strip indentation
-        size_t skip = std::min(min_indent, line.size());
-        result += line.substr(skip);
+        // Strip indentation, being careful about \x01 markers
+        size_t skip = 0;
+        size_t pos = 0;
+        while (skip < min_indent && pos < line.size()) {
+          if (line[pos] == '\x01') {
+            // Hit preserved whitespace marker - don't strip any more
+            break;
+          }
+          skip++;
+          pos++;
+        }
+
+        // Add the rest of the line, removing \x01 markers
+        for (size_t j = pos; j < line.size(); j++) {
+          if (line[j] != '\x01') {
+            result += line[j];
+          }
+        }
+
         if (i + 1 < lines.size())
           result += '\n';
       }
@@ -1025,11 +1069,11 @@ public:
             // inherit (expr) x → x = expr.x
             auto select = std::make_shared<Node>(
                 SelectNode(source, std::make_shared<Node>(ConstStringNode(name.value))));
-            attrs.attrs.push_back({name.value, select});
+            attrs.attrs.push_back(AttrBinding(name.value, select));
           } else {
             // inherit x → x = x
             auto var = std::make_shared<Node>(VarNode(0, name.value));
-            attrs.attrs.push_back({name.value, var});
+            attrs.attrs.push_back(AttrBinding(name.value, var));
           }
         }
 
@@ -1046,11 +1090,8 @@ public:
 
         if (consume(Token::EQUALS)) {
           auto value = parse_expr();
-          // For dynamic attrs, we use special marker in key and store expr as value
-          // This will need runtime evaluation - store as special node
-          // For now, convert to string at parse time if possible
-          // TODO: Full dynamic attr support needs IR node for dynamic keys
-          attrs.attrs.push_back({"__dynamic__", value});
+          // Dynamic attribute - name is evaluated at runtime
+          attrs.attrs.push_back(AttrBinding(name_expr, value));
         }
       } else if (current().type == Token::IDENT || current().type == Token::STRING ||
                  current().type == Token::INDENTED_STRING) {
@@ -1076,22 +1117,22 @@ public:
           // Desugar nested paths: a.b.c = v becomes a = { b = { c = v; }; }
           if (path.size() == 1) {
             // Simple case: just one key
-            attrs.attrs.push_back({path[0], value});
+            attrs.attrs.push_back(AttrBinding(path[0], value));
           } else {
             // Nested case: build nested attrsets from right to left
             auto nested = value;
             for (int i = path.size() - 1; i > 0; i--) {
               auto inner_attrs = AttrsetNode(false);
-              inner_attrs.attrs.push_back({path[i], nested});
+              inner_attrs.attrs.push_back(AttrBinding(path[i], nested));
               nested = std::make_shared<Node>(std::move(inner_attrs));
             }
-            attrs.attrs.push_back({path[0], nested});
+            attrs.attrs.push_back(AttrBinding(path[0], nested));
           }
         } else if (consume(Token::AT)) {
           // @ pattern - not affected by nested paths
           auto pattern = parse_expr();
           auto value = parse_expr();
-          attrs.attrs.push_back({path[0], value});
+          attrs.attrs.push_back(AttrBinding(path[0], value));
         }
       }
 
@@ -1111,27 +1152,25 @@ public:
   }
 
   std::shared_ptr<Node> parse_list() {
-    std::shared_ptr<Node> list = std::make_shared<Node>(ConstNullNode());
+    std::vector<std::shared_ptr<Node>> elements;
 
     if (consume(Token::RBRACKET)) {
-      return list;
+      return std::make_shared<Node>(ListNode(elements));
     }
 
-    std::vector<std::shared_ptr<Node>> elements;
     while (current().type != Token::RBRACKET) {
       elements.push_back(parse_expr());
-      if (!consume(Token::COMMA))
-        break;
-    }
-    expect(Token::RBRACKET);
-
-    for (auto it = elements.rbegin(); it != elements.rend(); ++it) {
-      list = std::make_shared<Node>(AppNode(
-          std::make_shared<Node>(AppNode(std::make_shared<Node>(VarNode(0, "__list")), *it)),
-          list));
+      if (!consume(Token::RBRACKET)) {
+        // Elements are whitespace-separated in Nix, no comma required
+        // But we'll continue parsing until we hit ]
+      } else {
+        // Found closing bracket
+        return std::make_shared<Node>(ListNode(elements));
+      }
     }
 
-    return list;
+    // Unreachable, but for safety
+    return std::make_shared<Node>(ListNode(elements));
   }
 
   void parse_bindings(std::vector<std::pair<std::string, std::shared_ptr<Node>>>& bindings) {

src/irc/serializer.cpp

@@ -64,6 +64,8 @@ struct Serializer::Impl {
       return NodeType::HAS_ATTR;
     if (node.holds<WithNode>())
       return NodeType::WITH;
+    if (node.holds<ListNode>())
+      return NodeType::LIST;
     if (node.holds<IfNode>())
       return NodeType::IF;
     if (node.holds<LetNode>())
@@ -129,10 +131,16 @@ struct Serializer::Impl {
     } else if (auto* n = node.get_if<AttrsetNode>()) {
       write_u8(n->recursive ? 1 : 0);
       write_u32(n->attrs.size());
-      for (const auto& [key, val] : n->attrs) {
-        write_string(key);
-        if (val)
-          write_node(*val);
+      for (const auto& binding : n->attrs) {
+        if (binding.is_dynamic()) {
+          write_u8(1); // Dynamic flag
+          write_node(*binding.dynamic_name);
+        } else {
+          write_u8(0); // Static flag
+          write_string(binding.static_name.value());
+        }
+        if (binding.value)
+          write_node(*binding.value);
       }
     } else if (auto* n = node.get_if<SelectNode>()) {
       if (n->expr)
@@ -155,6 +163,12 @@ struct Serializer::Impl {
         write_node(*n->attrs);
       if (n->body)
         write_node(*n->body);
+    } else if (auto* n = node.get_if<ListNode>()) {
+      write_u32(n->elements.size());
+      for (const auto& elem : n->elements) {
+        if (elem)
+          write_node(*elem);
+      }
     } else if (auto* n = node.get_if<IfNode>()) {
       if (n->cond)
         write_node(*n->cond);
@@ -335,9 +349,16 @@ struct Deserializer::Impl {
       uint32_t num_attrs = read_u32();
       AttrsetNode attrs(recursive, line);
       for (uint32_t i = 0; i < num_attrs; i++) {
-        std::string key = read_string();
-        auto val = read_node();
-        attrs.attrs.push_back({key, val});
+        uint8_t is_dynamic = read_u8();
+        if (is_dynamic) {
+          auto key_expr = read_node();
+          auto val = read_node();
+          attrs.attrs.push_back(AttrBinding(key_expr, val));
+        } else {
+          std::string key = read_string();
+          auto val = read_node();
+          attrs.attrs.push_back(AttrBinding(key, val));
+        }
       }
       return std::make_shared<Node>(std::move(attrs));
     }
@@ -363,6 +384,15 @@ struct Deserializer::Impl {
       auto body = read_node();
       return std::make_shared<Node>(WithNode(attrs, body, line));
     }
+    case NodeType::LIST: {
+      uint32_t num_elements = read_u32();
+      std::vector<std::shared_ptr<Node>> elements;
+      elements.reserve(num_elements);
+for (uint32_t i = 0; i < num_elements; i++) {
+        elements.push_back(read_node());
+      }
+      return std::make_shared<Node>(ListNode(std::move(elements), line));
+    }
     case NodeType::IF: {
       auto cond = read_node();
       auto then_branch = read_node();

src/irc/types.h

@@ -34,6 +34,7 @@ enum class NodeType : uint8_t {
   SELECT = 0x31,
   HAS_ATTR = 0x34,
   WITH = 0x32,
+  LIST = 0x33,
   IF = 0x40,
   LET = 0x50,
   LETREC = 0x51,
@@ -152,8 +153,24 @@ struct UnaryOpNode {
   UnaryOpNode(UnaryOp o, std::shared_ptr<Node> operand, uint32_t l = 0);
 };
 
+struct AttrBinding {
+  std::optional<std::string> static_name; // Static key like "foo"
+  std::shared_ptr<Node> dynamic_name;     // Dynamic key like ${expr}
+  std::shared_ptr<Node> value;
+
+  // Static attribute
+  AttrBinding(std::string name, std::shared_ptr<Node> val)
+      : static_name(std::move(name)), value(std::move(val)) {}
+
+  // Dynamic attribute
+  AttrBinding(std::shared_ptr<Node> name_expr, std::shared_ptr<Node> val)
+      : dynamic_name(std::move(name_expr)), value(std::move(val)) {}
+
+  bool is_dynamic() const { return !static_name.has_value(); }
+};
+
 struct AttrsetNode {
-  std::vector<std::pair<std::string, std::shared_ptr<Node>>> attrs;
+  std::vector<AttrBinding> attrs;
   bool recursive = false;
   uint32_t line = 0;
   AttrsetNode(bool rec = false, uint32_t l = 0) : recursive(rec), line(l) {}
@@ -228,14 +245,21 @@ struct ForceNode {
   ForceNode(std::shared_ptr<Node> e, uint32_t l = 0);
 };
 
+struct ListNode {
+  std::vector<std::shared_ptr<Node>> elements;
+  uint32_t line = 0;
+  ListNode(std::vector<std::shared_ptr<Node>> elems = {}, uint32_t l = 0)
+      : elements(std::move(elems)), line(l) {}
+};
+
 // Node wraps a variant for type-safe AST
 class Node {
 public:
-  using Variant =
-      std::variant<ConstIntNode, ConstFloatNode, ConstStringNode, ConstPathNode, ConstBoolNode,
-                   ConstNullNode, ConstURINode, ConstLookupPathNode, VarNode, LambdaNode, AppNode,
-                   BinaryOpNode, UnaryOpNode, ImportNode, AttrsetNode, SelectNode, HasAttrNode,
-                   WithNode, IfNode, LetNode, LetRecNode, AssertNode, ThunkNode, ForceNode>;
+  using Variant = std::variant<ConstIntNode, ConstFloatNode, ConstStringNode, ConstPathNode,
+                               ConstBoolNode, ConstNullNode, ConstURINode, ConstLookupPathNode,
+                               VarNode, LambdaNode, AppNode, BinaryOpNode, UnaryOpNode, ImportNode,
+                               AttrsetNode, SelectNode, HasAttrNode, WithNode, IfNode, LetNode,
+                               LetRecNode, AssertNode, ThunkNode, ForceNode, ListNode>;
 
   Variant data;