Symbolgraph extension symbols (#1372)

Support the `-emit-extension-block-symbols` mode which is used by default by SPM/Xcode since Swift 5.9.

CHANGELOG.md

@@ -6,7 +6,9 @@
 
 ##### Enhancements
 
-* None.
+* Support Swift 5.9 symbolgraph extension symbols.  
+  [John Fairhurst](https://github.com/johnfairh)
+  [#1368](https://github.com/realm/jazzy/issues/1368)
 
 ##### Bug Fixes
 

lib/jazzy/symbol_graph.rb

@@ -7,6 +7,7 @@
 require 'jazzy/symbol_graph/relationship'
 require 'jazzy/symbol_graph/sym_node'
 require 'jazzy/symbol_graph/ext_node'
+require 'jazzy/symbol_graph/ext_key'
 
 # This is the top-level symbolgraph driver that deals with
 # figuring out arguments, running the tool, and loading the
@@ -72,10 +73,12 @@ def self.parse_symbols(directory)
         # The @ part is for extensions in our module (before the @)
         # of types in another module (after the @).
         File.basename(filename) =~ /(.*?)(@(.*?))?\.symbols/
-        module_name = Regexp.last_match[3] || Regexp.last_match[1]
+        module_name = Regexp.last_match[1]
+        ext_module_name = Regexp.last_match[3] || module_name
+        json = File.read(filename)
         {
           filename =>
-            Graph.new(File.read(filename), module_name).to_sourcekit,
+            Graph.new(json, module_name, ext_module_name).to_sourcekit,
         }
       end.to_json
     end

lib/jazzy/symbol_graph/ext_key.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Jazzy
+  module SymbolGraph
+    # An ExtKey identifies an extension of a type, made up of the USR of
+    # the type and the constraints of the extension.  With Swift 5.9 extension
+    # symbols, the USR is the 'fake' USR invented by symbolgraph to solve the
+    # same problem as this type, which means less merging takes place.
+    class ExtKey
+      attr_accessor :usr
+      attr_accessor :constraints_text
+
+      def initialize(usr, constraints)
+        self.usr = usr
+        self.constraints_text = constraints.map(&:to_swift).join
+      end
+
+      def hash_key
+        usr + constraints_text
+      end
+
+      def eql?(other)
+        hash_key == other.hash_key
+      end
+
+      def hash
+        hash_key.hash
+      end
+    end
+
+    class ExtSymNode
+      def ext_key
+        ExtKey.new(usr, all_constraints.ext)
+      end
+    end
+  end
+end

lib/jazzy/symbol_graph/ext_node.rb

@@ -23,6 +23,7 @@ def initialize(type_constraints, ext_constraints)
     # an extension that we fabricate to resolve certain relationships.
     class ExtNode < BaseNode
       attr_accessor :usr
+      attr_accessor :real_usr
       attr_accessor :name
       attr_accessor :all_constraints # ExtConstraints
       attr_accessor :conformances # array, can be empty
@@ -75,15 +76,15 @@ def full_declaration
         decl + all_constraints.ext.to_where_clause
       end
 
-      def to_sourcekit(module_name)
+      def to_sourcekit(module_name, ext_module_name)
         declaration = full_declaration
         xml_declaration = "<swift>#{CGI.escapeHTML(declaration)}</swift>"
 
         hash = {
           'key.kind' => 'source.lang.swift.decl.extension',
-          'key.usr' => usr,
+          'key.usr' => real_usr || usr,
           'key.name' => name,
-          'key.modulename' => module_name,
+          'key.modulename' => ext_module_name,
           'key.parsed_declaration' => declaration,
           'key.annotated_decl' => xml_declaration,
         }
@@ -94,9 +95,7 @@ def to_sourcekit(module_name)
           end
         end
 
-        unless children.empty?
-          hash['key.substructure'] = children_to_sourcekit
-        end
+        add_children_to_sourcekit(hash, module_name)
 
         hash
       end
@@ -112,5 +111,23 @@ def <=>(other)
         sort_key <=> other.sort_key
       end
     end
+
+    # An ExtSymNode is an extension generated from a Swift 5.9 extension
+    # symbol, for extensions of types from other modules only.
+    class ExtSymNode < ExtNode
+      attr_accessor :symbol
+
+      def initialize(symbol)
+        self.symbol = symbol
+        super(symbol.usr, symbol.full_name,
+              # sadly can't tell what constraints are inherited vs added
+              ExtConstraints.new([], symbol.constraints))
+      end
+
+      def to_sourcekit(module_name, ext_module_name)
+        hash = super
+        symbol.add_to_sourcekit(hash)
+      end
+    end
   end
 end

lib/jazzy/symbol_graph/graph.rb

@@ -7,37 +7,40 @@ module SymbolGraph
     # Deserialize it to Symbols and Relationships, then rebuild
     # the AST shape using SymNodes and ExtNodes and extract SourceKit json.
     class Graph
-      attr_accessor :module_name
+      attr_accessor :module_name # Our module
+      attr_accessor :ext_module_name # Module being extended
       attr_accessor :symbol_nodes # usr -> SymNode
       attr_accessor :relationships # [Relationship]
       attr_accessor :ext_nodes # (usr, constraints) -> ExtNode
 
       # Parse the JSON into flat tables of data
-      def initialize(json, module_name)
+      def initialize(json, module_name, ext_module_name)
         self.module_name = module_name
+        self.ext_module_name = ext_module_name
         graph = JSON.parse(json, symbolize_names: true)
 
         self.symbol_nodes = {}
+        self.ext_nodes = {}
+
         graph[:symbols].each do |hash|
           symbol = Symbol.new(hash)
-          symbol_nodes[symbol.usr] = SymNode.new(symbol)
+          if symbol.extension?
+            node = ExtSymNode.new(symbol)
+            ext_nodes[node.ext_key] = node
+          else
+            symbol_nodes[symbol.usr] = SymNode.new(symbol)
+          end
         end
 
         self.relationships =
           graph[:relationships].map { |hash| Relationship.new(hash) }
-
-        self.ext_nodes = {}
       end
 
-      # ExtNode index.  (type USR, extension constraints) -> ExtNode.
+      # ExtNode index.  ExtKey (type USR, extension constraints) -> ExtNode.
       # This minimizes the number of extensions
 
-      def ext_key(usr, constraints)
-        usr + constraints.map(&:to_swift).join
-      end
-
       def add_ext_member(type_usr, member_node, constraints)
-        key = ext_key(type_usr, constraints.ext)
+        key = ExtKey.new(type_usr, constraints.ext)
         if ext_node = ext_nodes[key]
           ext_node.add_child(member_node)
         else
@@ -50,7 +53,7 @@ def add_ext_conformance(type_usr,
                               type_name,
                               protocol,
                               constraints)
-        key = ext_key(type_usr, constraints.ext)
+        key = ExtKey.new(type_usr, constraints.ext)
         if ext_node = ext_nodes[key]
           ext_node.add_conformance(protocol)
         else
@@ -149,6 +152,15 @@ def rebuild_inherits(_rel, source, target)
         end
       end
 
+      # "References to fake_usr should be real_usr"
+      def unalias_extensions(fake_usr, real_usr)
+        ext_nodes.each_pair do |key, ext|
+          if key.usr == fake_usr
+            ext.real_usr = real_usr
+          end
+        end
+      end
+
       # Process a structural relationship to link nodes
       def rebuild_rel(rel)
         source = symbol_nodes[rel.source_usr]
@@ -166,29 +178,29 @@ def rebuild_rel(rel)
 
         when :inheritsFrom
           rebuild_inherits(rel, source, target)
+
+        when :extensionTo
+          unalias_extensions(rel.source_usr, rel.target_usr)
         end
+
         # don't seem to care about:
         # - overrides: not bothered, also unimplemented for protocols
       end
 
       # Rebuild the AST structure  and convert to SourceKit
       def to_sourcekit
-        # Do default impls after the others so we can find protocol
-        # type nodes from protocol requirements.
-        default_impls, other_rels =
-          relationships.partition(&:default_implementation?)
-        (other_rels + default_impls).each { |r| rebuild_rel(r) }
+        relationships.sort.each { |r| rebuild_rel(r) }
 
         root_symbol_nodes =
           symbol_nodes.values
             .select(&:top_level_decl?)
             .sort
-            .map(&:to_sourcekit)
+            .map { |n| n.to_sourcekit(module_name) }
 
         root_ext_nodes =
           ext_nodes.values
             .sort
-            .map { |n| n.to_sourcekit(module_name) }
+            .map { |n| n.to_sourcekit(module_name, ext_module_name) }
         {
           'key.diagnostic_stage' => 'parse',
           'key.substructure' => root_symbol_nodes + root_ext_nodes,

lib/jazzy/symbol_graph/relationship.rb

@@ -10,9 +10,14 @@ class Relationship
       attr_accessor :target_fallback # can be nil
       attr_accessor :constraints # array, can be empty
 
-      KINDS = %w[memberOf conformsTo defaultImplementationOf
-                 overrides inheritsFrom requirementOf
-                 optionalRequirementOf].freeze
+      # Order matters: defaultImplementationOf after the protocols
+      # have been defined; extensionTo after all the extensions have
+      # been discovered.
+      KINDS = %w[memberOf conformsTo overrides inheritsFrom
+                 requirementOf optionalRequirementOf
+                 defaultImplementationOf extensionTo].freeze
+
+      KINDS_INDEX = KINDS.to_h { |i| [i.to_sym, KINDS.index(i)] }.freeze
 
       def protocol_requirement?
         %i[requirementOf optionalRequirementOf].include? kind
@@ -22,6 +27,10 @@ def default_implementation?
         kind == :defaultImplementationOf
       end
 
+      def extension_to?
+        kind == :extensionTo
+      end
+
       # Protocol conformances added by compiler to actor decls that
       # users aren't interested in.
       def actor_protocol?
@@ -43,6 +52,15 @@ def initialize(hash)
         end
         self.constraints = Constraint.new_list(hash[:swiftConstraints] || [])
       end
+
+      # Sort order
+      include Comparable
+
+      def <=>(other)
+        return 0 if kind == other.kind
+
+        KINDS_INDEX[kind] <=> KINDS_INDEX[other.kind]
+      end
     end
   end
 end

lib/jazzy/symbol_graph/sym_node.rb

@@ -18,14 +18,16 @@ def add_child(child)
         children.append(child)
       end
 
-      def children_to_sourcekit
-        children.sort.map(&:to_sourcekit)
+      def add_children_to_sourcekit(hash, module_name)
+        unless children.empty?
+          hash['key.substructure'] =
+            children.sort.map { |c| c.to_sourcekit(module_name) }
+        end
       end
     end
 
     # A SymNode is a node of the reconstructed syntax tree holding a symbol.
     # It can turn itself into SourceKit and helps decode extensions.
-    # rubocop:disable Metrics/ClassLength
     class SymNode < BaseNode
       attr_accessor :symbol
       attr_writer :override
@@ -128,41 +130,28 @@ def full_declaration
           .join("\n")
       end
 
-      # rubocop:disable Metrics/MethodLength
-      def to_sourcekit
+      def to_sourcekit(module_name)
         declaration = full_declaration
         xml_declaration = "<swift>#{CGI.escapeHTML(declaration)}</swift>"
 
         hash = {
           'key.kind' => symbol.kind,
           'key.usr' => symbol.usr,
           'key.name' => symbol.name,
-          'key.accessibility' => symbol.acl,
-          'key.parsed_decl' => declaration,
+          'key.modulename' => module_name,
+          'key.parsed_declaration' => declaration,
           'key.annotated_decl' => xml_declaration,
           'key.symgraph_async' => async?,
         }
-        if docs = symbol.doc_comments
-          hash['key.doc.comment'] = docs
-          hash['key.doc.full_as_xml'] = ''
-        end
         if params = symbol.parameter_names
           hash['key.doc.parameters'] =
             params.map { |name| { 'name' => name } }
         end
-        if location = symbol.location
-          hash['key.filepath'] = location[:filename]
-          hash['key.doc.line'] = location[:line] + 1
-          hash['key.doc.column'] = location[:character] + 1
-        end
-        unless children.empty?
-          hash['key.substructure'] = children_to_sourcekit
-        end
         hash['key.symgraph_spi'] = true if symbol.spi
 
-        hash
+        add_children_to_sourcekit(hash, module_name)
+        symbol.add_to_sourcekit(hash)
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Sort order - by symbol
       include Comparable
@@ -171,6 +160,5 @@ def <=>(other)
         symbol <=> other.symbol
       end
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end