Add a rubocop ast translator

Author: kddnewton

lib/syntax_tree/parser.rb

@@ -275,7 +275,7 @@ def find_keyword(name)
     end
 
     def find_keyword_between(name, left, right)
-      bounds = left.location.end_char...right.location.start_char
+      bounds = left.end_char...right.start_char
       index =
         tokens.rindex do |token|
           char = token.location.start_char
@@ -1807,19 +1807,19 @@ def on_for(index, collection, statements)
       in_keyword = consume_keyword(:in)
       ending = consume_keyword(:end)
 
-      # Consume the do keyword if it exists so that it doesn't get confused for
-      # some other block
-      if (keyword = find_keyword_between(:do, collection, ending))
-        tokens.delete(keyword)
-      end
+      delimiter =
+        find_keyword_between(:do, collection, ending) ||
+          find_token_between(Semicolon, collection, ending)
+
+      tokens.delete(delimiter) if delimiter
 
       start_char =
-        find_next_statement_start((keyword || collection).location.end_char)
+        find_next_statement_start((delimiter || collection).location.end_char)
 
       statements.bind(
         start_char,
         start_char -
-          line_counts[(keyword || collection).location.end_line - 1].start,
+          line_counts[(delimiter || collection).location.end_line - 1].start,
         ending.location.start_char,
         ending.location.start_column
       )
@@ -3328,10 +3328,13 @@ def on_sclass(target, bodystmt)
       )
     end
 
+    # Semicolons are tokens that get added to the token list but never get
+    # attached to the AST. Because of this they only need to track their
+    # associated location so they can be used for computing bounds.
     class Semicolon
       attr_reader :location
 
-      def initialize(location:)
+      def initialize(location)
         @location = location
       end
     end
@@ -3340,13 +3343,12 @@ def initialize(location:)
     #   on_semicolon: (String value) -> Semicolon
     def on_semicolon(value)
       tokens << Semicolon.new(
-        location:
-          Location.token(
-            line: lineno,
-            char: char_pos,
-            column: current_column,
-            size: value.size
-          )
+        Location.token(
+          line: lineno,
+          char: char_pos,
+          column: current_column,
+          size: value.size
+        )
       )
     end
 

lib/syntax_tree/translation.rb

@@ -13,5 +13,16 @@ def self.to_parser(node, buffer)
 
       node.accept(Parser.new(buffer))
     end
+
+    # This method translates the given node into the representation defined by
+    # the rubocop/rubocop-ast gem. We don't explicitly list it as a dependency
+    # because it's not required for the core functionality of Syntax Tree.
+    def self.to_rubocop_ast(node, buffer)
+      require "rubocop/ast"
+      require_relative "translation/parser"
+      require_relative "translation/rubocop_ast"
+
+      node.accept(RuboCopAST.new(buffer))
+    end
   end
 end

lib/syntax_tree/translation/parser.rb

@@ -5,6 +5,76 @@ module Translation
     # This visitor is responsible for converting the syntax tree produced by
     # Syntax Tree into the syntax tree produced by the whitequark/parser gem.
     class Parser < BasicVisitor
+      # Heredocs are represented _very_ differently in the parser gem from how
+      # they are represented in the Syntax Tree AST. This class is responsible
+      # for handling the translation.
+      class HeredocBuilder
+        Line = Struct.new(:value, :segments)
+
+        attr_reader :node, :segments
+
+        def initialize(node)
+          @node = node
+          @segments = []
+        end
+
+        def <<(segment)
+          if segment.type == :str && segments.last &&
+               segments.last.type == :str &&
+               !segments.last.children.first.end_with?("\n")
+            segments.last.children.first << segment.children.first
+          else
+            segments << segment
+          end
+        end
+
+        def trim!
+          return unless node.beginning.value[2] == "~"
+          lines = [Line.new(+"", [])]
+
+          segments.each do |segment|
+            lines.last.segments << segment
+
+            if segment.type == :str
+              lines.last.value << segment.children.first
+
+              if lines.last.value.end_with?("\n")
+                lines << Line.new(+"", [])
+              end
+            end
+          end
+
+          lines.pop if lines.last.value.empty?
+          return if lines.empty?
+
+          segments.clear
+          lines.each do |line|
+            remaining = node.dedent
+
+            line.segments.each do |segment|
+              if segment.type == :str
+                if remaining > 0
+                  whitespace = segment.children.first[/^\s{0,#{remaining}}/]
+                  segment.children.first.sub!(/^#{whitespace}/, "")
+                  remaining -= whitespace.length
+                end
+
+                if node.beginning.value[3] != "'" && segments.any? &&
+                     segments.last.type == :str &&
+                     segments.last.children.first.end_with?("\\\n")
+                  segments.last.children.first.gsub!(/\\\n\z/, "")
+                  segments.last.children.first.concat(segment.children.first)
+                elsif !segment.children.first.empty?
+                  segments << segment
+                end
+              else
+                segments << segment
+              end
+            end
+          end
+        end
+      end
+
       attr_reader :buffer, :stack
 
       def initialize(buffer)
@@ -665,6 +735,25 @@ def visit_command_call(node)
             node.end_char
           end
 
+        expression =
+          if node.arguments.is_a?(ArgParen)
+            srange(node.start_char, node.arguments.end_char)
+          elsif node.arguments.is_a?(Args) && node.arguments.parts.any?
+            last_part = node.arguments.parts.last
+            end_char =
+              if last_part.is_a?(Heredoc)
+                last_part.beginning.end_char
+              else
+                last_part.end_char
+              end
+
+            srange(node.start_char, end_char)
+          elsif node.block
+            srange_node(node.message)
+          else
+            srange_node(node)
+          end
+
         call =
           s(
             if node.operator.is_a?(Op) && node.operator.value == "&."
@@ -690,14 +779,7 @@ def visit_command_call(node)
               node.message == :call ? nil : srange_node(node.message),
               begin_token,
               end_token,
-              if node.arguments.is_a?(ArgParen) ||
-                   (node.arguments.is_a?(Args) && node.arguments.parts.any?)
-                srange(node.start_char, node.arguments.end_char)
-              elsif node.block
-                srange_node(node.message)
-              else
-                srange_node(node)
-              end
+              expression
             )
           )
 
@@ -1049,7 +1131,8 @@ def visit_for(node)
           smap_for(
             srange_length(node.start_char, 3),
             srange_find_between(node.index, node.collection, "in"),
-            srange_search_between(node.collection, node.statements, "do"),
+            srange_search_between(node.collection, node.statements, "do") ||
+              srange_search_between(node.collection, node.statements, ";"),
             srange_length(node.end_char, -3),
             srange_node(node)
           )
@@ -1078,98 +1161,43 @@ def visit_hash(node)
         )
       end
 
-      # Heredocs are represented _very_ differently in the parser gem from how
-      # they are represented in the Syntax Tree AST. This class is responsible
-      # for handling the translation.
-      class HeredocSegments
-        HeredocLine = Struct.new(:value, :segments)
-
-        attr_reader :node, :segments
-
-        def initialize(node)
-          @node = node
-          @segments = []
-        end
-
-        def <<(segment)
-          if segment.type == :str && segments.last &&
-               segments.last.type == :str &&
-               !segments.last.children.first.end_with?("\n")
-            segments.last.children.first << segment.children.first
-          else
-            segments << segment
-          end
-        end
-
-        def trim!
-          return unless node.beginning.value[2] == "~"
-          lines = [HeredocLine.new(+"", [])]
-
-          segments.each do |segment|
-            lines.last.segments << segment
-
-            if segment.type == :str
-              lines.last.value << segment.children.first
-
-              if lines.last.value.end_with?("\n")
-                lines << HeredocLine.new(+"", [])
-              end
-            end
-          end
-
-          lines.pop if lines.last.value.empty?
-          return if lines.empty?
-
-          segments.clear
-          lines.each do |line|
-            remaining = node.dedent
-
-            line.segments.each do |segment|
-              if segment.type == :str
-                if remaining > 0
-                  whitespace = segment.children.first[/^\s{0,#{remaining}}/]
-                  segment.children.first.sub!(/^#{whitespace}/, "")
-                  remaining -= whitespace.length
-                end
-
-                if node.beginning.value[3] != "'" && segments.any? &&
-                     segments.last.type == :str &&
-                     segments.last.children.first.end_with?("\\\n")
-                  segments.last.children.first.gsub!(/\\\n\z/, "")
-                  segments.last.children.first.concat(segment.children.first)
-                elsif !segment.children.first.empty?
-                  segments << segment
-                end
-              else
-                segments << segment
-              end
-            end
-          end
-        end
-      end
-
       # Visit a Heredoc node.
       def visit_heredoc(node)
-        heredoc_segments = HeredocSegments.new(node)
+        heredoc = HeredocBuilder.new(node)
 
+        # For each part of the heredoc, if it's a string content node, split it
+        # into multiple string content nodes, one for each line. Otherwise,
+        # visit the node as normal.
         node.parts.each do |part|
           if part.is_a?(TStringContent) && part.value.count("\n") > 1
-            part
-              .value
-              .split("\n")
-              .each { |line| heredoc_segments << s(:str, ["#{line}\n"], nil) }
+            index = part.start_char
+            lines = part.value.split("\n")
+
+            lines.each do |line|
+              length = line.length + 1
+              location = smap_collection_bare(srange_length(index, length))
+
+              heredoc << s(:str, ["#{line}\n"], location)
+              index += length
+            end
           else
-            heredoc_segments << visit(part)
+            heredoc << visit(part)
           end
         end
 
-        heredoc_segments.trim!
+        # Now that we have all of the pieces on the heredoc, we can trim it if
+        # it is a heredoc that supports trimming (i.e., it has a ~ on the
+        # declaration).
+        heredoc.trim!
+
+        # Generate the location for the heredoc, which goes from the declaration
+        # to the ending delimiter.
         location =
           smap_heredoc(
             srange_node(node.beginning),
             srange(
               if node.parts.empty?
-                node.beginning.end_char
+                node.beginning.end_char + 1
               else
                 node.parts.first.start_char
               end,
@@ -1178,15 +1206,15 @@ def visit_heredoc(node)
             srange(node.ending.start_char, node.ending.end_char - 1)
           )
 
+        # Finally, decide which kind of heredoc node to generate based on its
+        # declaration and contents.
         if node.beginning.value.match?(/`\w+`\z/)
-          s(:xstr, heredoc_segments.segments, location)
-        elsif heredoc_segments.segments.length > 1
-          s(:dstr, heredoc_segments.segments, location)
-        elsif heredoc_segments.segments.empty?
-          s(:dstr, [], location)
-        else
-          segment = heredoc_segments.segments.first
+          s(:xstr, heredoc.segments, location)
+        elsif heredoc.segments.length == 1
+          segment = heredoc.segments.first
           s(segment.type, segment.children, location)
+        else
+          s(:dstr, heredoc.segments, location)
         end
       end
 

lib/syntax_tree/translation/rubocop_ast.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SyntaxTree
+  module Translation
+    # This visitor is responsible for converting the syntax tree produced by
+    # Syntax Tree into the syntax tree produced by the rubocop/rubocop-ast gem.
+    class RuboCopAST < Parser
+      private
+
+      # This method is effectively the same thing as the parser gem except that
+      # it uses the rubocop-ast specializations of the nodes.
+      def s(type, children, location)
+        ::RuboCop::AST::Builder::NODE_MAP.fetch(type, ::RuboCop::AST::Node).new(
+          type,
+          children,
+          location: location
+        )
+      end
+    end
+  end
+end