doc comments and annotations in Souffle datalog (#2472)

* doc comments and annotations in Souffle datalog

Annotations are free meta-data that can be attached to most items of a
Souffle datalog program.

A documentation comment is a syntactic sugar for a `doc` annotation.

The design is close to Rust's *attributes*.

All program's and component's items can have *outer* annotations of
the form `@[annotation]` or `/// doc comment`. Attributes and ADT
branches also accept outer annotations.

Some items can have *inner* annotations of the for `@![annotation]`
or `//! doc comment`:
- after `{` of a component, before the first component's item.
- after `{` of an ADT branch, before the first attribute of the branch.
- after `:-` of a clause, before the first constraint.

Annotations should not appear before `.include` or before any `}` or
before the end of a file.

An annotation's payload may be of these forms:
- an identifier: `@[IDENT]`
- possibly followed by delimited token stream: `@[IDENT DELIM]`
- or `=` and a delimited token stream: `@[IDENT = DELIM]` or
  `@[IDENT = TOKEN TT*]`

Where:
- `IDENT` is an identifier or a keyword of Souffle.
- `TOKEN` is any token except delimiters.
- `TS ::= TT+` is a token-stream.
- `TT ::= TOKEN | DELIM` is a token-tree.
- `DELIM ::= '(' TS? ')' | '[' TS? ']' | '{' TS? '}'` is a delimited
  token-stream.

Use cases:
- documentation comments attached to AST nodes ease the writing of documentation generation tools.
- annotations could replace "dirty" parts of the current Souffle grammar (stateful, choice-domain, .plan ...).
- annotations brings extensibility without modifications of the core grammar.

src/CMakeLists.txt

@@ -9,6 +9,7 @@ set(SOUFFLE_SOURCES
     FunctorOps.cpp
     Global.cpp
     MainDriver.cpp
+    ast/Annotation.cpp
     ast/Aggregator.cpp
     ast/IntrinsicAggregator.cpp
     ast/UserDefinedAggregator.cpp
@@ -48,6 +49,7 @@ set(SOUFFLE_SOURCES
     ast/StringConstant.cpp
     ast/SubsetType.cpp
     ast/Term.cpp
+    ast/TokenTree.cpp
     ast/TranslationUnit.cpp
     ast/Type.cpp
     ast/TypeCast.cpp

src/ast/AlgebraicDataType.cpp

@@ -31,6 +31,7 @@ Node::NodeVec AlgebraicDataType::getChildren() const {
 }
 
 void AlgebraicDataType::print(std::ostream& os) const {
+    printAnnotations(os);
     os << tfm::format(".type %s = %s", getQualifiedName(), join(branches, " | "));
 }
 

src/ast/Annotation.cpp

@@ -0,0 +1,65 @@
+/*
+ * Souffle - A Datalog Compiler
+ * Copyright (c) 2022, The Souffle Developers. All rights reserved
+ * Licensed under the Universal Permissive License v 1.0 as shown at:
+ * - https://opensource.org/licenses/UPL
+ * - <souffle root>/licenses/SOUFFLE-UPL.txt
+ */
+#include "Annotation.h"
+
+#include <cassert>
+
+namespace souffle::ast {
+
+Annotation::Annotation() : kind(), style(), label(), tokens(), location() {}
+
+Annotation::Annotation(Kind k, Style sty, QualifiedName lbl, TokenStream ts, SrcLocation loc)
+        : kind(k), style(sty), label(lbl), tokens(ts), location(loc) {}
+
+bool Annotation::operator==(const Annotation& other) const {
+    return (kind == other.kind) && (style == other.style) && (label == other.label) &&
+           (tokens == other.tokens);
+}
+
+void Annotation::print(std::ostream& os) const {
+    os << "@";
+    if (style == Style::Inner) {
+        os << "!";
+    }
+    os << "[" << label;
+    if (!tokens.empty()) {
+        os << " ";
+    }
+    printTokenStream(os, tokens);
+    os << "]";
+}
+
+void Annotation::printAsOuter(std::ostream& os) const {
+    os << "@[" << label;
+    if (!tokens.empty()) {
+        os << " ";
+    }
+    printTokenStream(os, tokens);
+    os << "]";
+}
+
+Annotation::Kind Annotation::getKind() const {
+    return kind;
+}
+
+Annotation::Style Annotation::getStyle() const {
+    return style;
+}
+
+const SrcLocation& Annotation::getSrcLoc() const {
+    return location;
+}
+
+const QualifiedName& Annotation::getLabel() const {
+    return label;
+}
+
+const TokenStream& Annotation::getTokens() const {
+    return tokens;
+}
+}  // namespace souffle::ast

src/ast/Annotation.h

@@ -0,0 +1,61 @@
+/*
+ * Souffle - A Datalog Compiler
+ * Copyright (c) 2022, The Souffle Developers. All rights reserved
+ * Licensed under the Universal Permissive License v 1.0 as shown at:
+ * - https://opensource.org/licenses/UPL
+ * - <souffle root>/licenses/SOUFFLE-UPL.txt
+ */
+#pragma once
+
+#include "QualifiedName.h"
+#include "TokenTree.h"
+#include "parser/SrcLocation.h"
+
+#include <list>
+
+namespace souffle::ast {
+
+class Annotation {
+public:
+    enum class Kind { Normal, DocComment };
+
+    enum class Style { Outer, Inner };
+
+    Annotation();
+    Annotation(Kind kind, Style style, QualifiedName lbl, TokenStream ts, SrcLocation loc = {});
+    Annotation(Annotation&&) = default;
+    Annotation(const Annotation&) = default;
+    Annotation& operator=(Annotation&&) = default;
+    Annotation& operator=(const Annotation&) = default;
+
+    bool operator==(const Annotation& other) const;
+
+    Kind getKind() const;
+
+    Style getStyle() const;
+
+    const SrcLocation& getSrcLoc() const;
+
+    const QualifiedName& getLabel() const;
+
+    const TokenStream& getTokens() const;
+
+    void print(std::ostream&) const;
+
+    void printAsOuter(std::ostream&) const;
+
+private:
+    Kind kind;
+
+    Style style;
+
+    QualifiedName label;
+
+    TokenStream tokens;
+
+    SrcLocation location;
+};
+
+using AnnotationList = std::list<Annotation>;
+
+}  // namespace souffle::ast

src/ast/Atom.cpp

@@ -53,6 +53,7 @@ Node::NodeVec Atom::getChildren() const {
 }
 
 void Atom::print(std::ostream& os) const {
+    printAnnotations(os);
     os << getQualifiedName() << "(" << join(arguments) << ")";
 }
 

src/ast/Attribute.cpp

@@ -25,6 +25,7 @@ void Attribute::setTypeName(QualifiedName name) {
 }
 
 void Attribute::print(std::ostream& os) const {
+    printAnnotations(os);
     os << name << ":" << typeName;
     if (isLattice) {
         os << "<>";

src/ast/BranchType.cpp

@@ -25,6 +25,7 @@ std::vector<Attribute*> BranchType::getFields() {
 }
 
 void BranchType::print(std::ostream& os) const {
+    printAnnotations(os);
     os << tfm::format("%s {%s}", name, join(fields, ", "));
 }
 
@@ -40,4 +41,8 @@ bool BranchType::classof(const Node* n) {
     return n->getKind() == NK_BranchType;
 }
 
+bool BranchType::equal(const Node& /* other */) const {
+    return false;
+}
+
 }  // namespace souffle::ast

src/ast/BranchType.h

@@ -64,6 +64,8 @@ class BranchType : public Node {
     void print(std::ostream& os) const override;
 
 private:
+    bool equal(const Node& /* other */) const override;
+
     BranchType* cloning() const override;
 
 private:

src/ast/Clause.cpp

@@ -74,6 +74,7 @@ Node::NodeVec Clause::getChildren() const {
 }
 
 void Clause::print(std::ostream& os) const {
+    printAnnotations(os);
     os << *head;
     if (!bodyLiterals.empty()) {
         os << " :- \n   " << join(bodyLiterals, ",\n   ");
@@ -84,21 +85,31 @@ void Clause::print(std::ostream& os) const {
     }
 }
 
+void Clause::printForDebugInfo(std::ostream& os) const {
+    os << *head;
+    if (!bodyLiterals.empty()) {
+        os << " :- \n   " << join(bodyLiterals, ",\n   ");
+    }
+    os << ".";
+}
+
 bool Clause::equal(const Node& node) const {
     const auto& other = asAssert<Clause>(node);
     return equal_ptr(head, other.head) && equal_targets(bodyLiterals, other.bodyLiterals) &&
            equal_ptr(plan, other.plan);
 }
 
 Clause* Clause::cloning() const {
-    return new Clause(clone(head), clone(bodyLiterals), clone(plan), getSrcLoc());
+    auto* cl = new Clause(clone(head), clone(bodyLiterals), clone(plan), getSrcLoc());
+    return cl;
 }
 
 Clause* Clause::cloneHead() const {
     Clause* myClone = new Clause(clone(head), getSrcLoc());
     if (getExecutionPlan() != nullptr) {
         myClone->setExecutionPlan(clone(getExecutionPlan()));
     }
+    myClone->setAnnotationsFrom(*this);
     return myClone;
 }
 

src/ast/Clause.h

@@ -84,6 +84,9 @@ class Clause : public Node {
 
     void apply(const NodeMapper& map) override;
 
+    /** Print this clause for `ram::DebugInfo` without annotations */
+    void printForDebugInfo(std::ostream& os) const;
+
     static bool classof(const Node*);
 
 protected:

src/ast/Component.cpp

@@ -130,6 +130,7 @@ void Component::print(std::ostream& os) const {
         os << prefix << join(xs, sep) << "\n";
     };
 
+    printAnnotations(os);
     os << ".comp " << *componentType << " ";
     show(baseComponents, ",", ": ");
     os << "{\n";
@@ -158,6 +159,7 @@ bool Component::equal(const Node& node) const {
             equal_targets(relations, other.relations) &&
             equal_targets(clauses, other.clauses) &&
             equal_targets(directives, other.directives) &&
+            equal_targets(lattices, other.lattices) &&
             overrideRules != other.overrideRules);
            // clang-format off
 }
@@ -172,6 +174,7 @@ Component* Component::cloning() const {
     res->relations = clone(relations);
     res->clauses = clone(clauses);
     res->directives = clone(directives);
+    res->lattices = clone(lattices);
     res->overrideRules = overrideRules;
     return res;
 }

src/ast/Component.h

@@ -20,6 +20,7 @@
 #include "ast/ComponentInit.h"
 #include "ast/ComponentType.h"
 #include "ast/Directive.h"
+#include "ast/ItemContainer.h"
 #include "ast/Lattice.h"
 #include "ast/Node.h"
 #include "ast/Relation.h"
@@ -43,7 +44,7 @@ namespace souffle::ast {
  *
  * Component consists of type declaration, relations, rules, etc.
  */
-class Component : public Node {
+class Component : public Node, public ItemContainer {
 public:
     Component(SrcLocation loc = {});
 
@@ -62,48 +63,48 @@ class Component : public Node {
     void addBaseComponent(Own<ComponentType> component);
 
     /** Add type */
-    void addType(Own<Type> t);
+    void addType(Own<Type> t) override;
 
     /** Get types */
-    std::vector<Type*> getTypes() const;
+    std::vector<Type*> getTypes() const override;
 
     /** Add lattice */
-    void addLattice(Own<Lattice> lat);
+    void addLattice(Own<Lattice> lat) override;
 
-    std::vector<Lattice*> getLattices() const;
+    std::vector<Lattice*> getLattices() const override;
 
     /** Copy base components */
     void copyBaseComponents(const Component& other);
 
     /** Add relation */
-    void addRelation(Own<Relation> r);
+    void addRelation(Own<Relation> r) override;
 
     /** Get relations */
-    std::vector<Relation*> getRelations() const;
+    std::vector<Relation*> getRelations() const override;
 
     /** Add clause */
-    void addClause(Own<Clause> c);
+    void addClause(Own<Clause> c) override;
 
     /** Get clauses */
-    std::vector<Clause*> getClauses() const;
+    std::vector<Clause*> getClauses() const override;
 
     /** Add directive */
-    void addDirective(Own<Directive> directive);
+    void addDirective(Own<Directive> directive) override;
 
     /** Get directive statements */
-    std::vector<Directive*> getDirectives() const;
+    std::vector<Directive*> getDirectives() const override;
 
     /** Add components */
-    void addComponent(Own<Component> c);
+    void addComponent(Own<Component> c) override;
 
     /** Get components */
-    std::vector<Component*> getComponents() const;
+    std::vector<Component*> getComponents() const override;
 
     /** Add instantiation */
-    void addInstantiation(Own<ComponentInit> i);
+    void addInstantiation(Own<ComponentInit> i) override;
 
     /** Get instantiation */
-    std::vector<ComponentInit*> getInstantiations() const;
+    std::vector<ComponentInit*> getInstantiations() const override;
 
     /** Add override */
     void addOverride(const std::string& name) {

src/ast/ComponentInit.cpp

@@ -38,6 +38,7 @@ Node::NodeVec ComponentInit::getChildren() const {
 }
 
 void ComponentInit::print(std::ostream& os) const {
+    printAnnotations(os);
     os << ".init " << instanceName << " = " << *componentType;
 }
 

src/ast/Counter.cpp

@@ -25,4 +25,8 @@ bool Counter::classof(const Node* n) {
     return n->getKind() == NK_Counter;
 }
 
+bool Counter::equal(const Node&) const {
+    return true;
+}
+
 }  // namespace souffle::ast

src/ast/Counter.h

@@ -37,6 +37,8 @@ class Counter : public Argument {
     void print(std::ostream& os) const override;
 
 private:
+    bool equal(const Node&) const override;
+
     Counter* cloning() const override;
 };
 

src/ast/Directive.cpp

@@ -37,6 +37,7 @@ void Directive::addParameter(const std::string& key, std::string value) {
 }
 
 void Directive::print(std::ostream& os) const {
+    printAnnotations(os);
     os << "." << type << " " << name;
     if (!parameters.empty()) {
         os << "(" << join(parameters, ",", [](std::ostream& out, const auto& arg) {

src/ast/FunctorDeclaration.cpp

@@ -28,11 +28,8 @@ FunctorDeclaration::FunctorDeclaration(
 }
 
 void FunctorDeclaration::print(std::ostream& out) const {
-    auto convert = [&](Own<Attribute> const& attr) {
-        return attr->getName() + ": " + attr->getTypeName().toString();
-    };
-
-    tfm::format(out, ".functor %s(%s): %s", name, join(map(params, convert), ","), returnType->getTypeName());
+    printAnnotations(out);
+    tfm::format(out, ".functor %s(%s): %s", name, join(params, ","), returnType->getTypeName());
     if (stateful) {
         out << " stateful";
     }