designate doc comments

Author: resolritter

corpus/source_files.txt

@@ -55,6 +55,33 @@ Line comments
 (source_file
   (line_comment))
 
+============================================
+Doc comments
+============================================
+
+/// Doc
+/// Comment
+// / Now a line comment (note the space separating the third slash)
+
+/// Doc
+/// Comment
+//// Four slashes makes the line a normal comment
+/// Doc comment got interrupted by the line above
+
+// Check that this comment does not eat the lines forward
+foo;
+
+----
+
+(source_file
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (line_comment)
+  (identifier))
+
 =====================================
 Greek letters in identifiers
 =====================================

grammar.js

@@ -38,13 +38,15 @@ const primitive_types = numeric_types.concat(['bool', 'str', 'char'])
 module.exports = grammar({
   name: 'rust',
 
-  extras: $ => [/\s/, $.line_comment, $.block_comment],
+  extras: $ => [/\s/, $.line_comment, $.block_comment, $.doc_comment],
 
   externals: $ => [
     $._string_content,
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
+    $.line_comment,
+    $.doc_comment
   ],
 
   supertypes: $ => [
@@ -1425,15 +1427,6 @@ module.exports = grammar({
 
     boolean_literal: $ => choice('true', 'false'),
 
-    comment: $ => choice(
-      $.line_comment,
-      $.block_comment
-    ),
-
-    line_comment: $ => token(seq(
-      '//', /.*/
-    )),
-
     _path: $ => choice(
       $.self,
       alias(choice(...primitive_types), $.identifier),

src/scanner.c

@@ -6,6 +6,8 @@ enum TokenType {
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
+  LINE_COMMENT,
+  DOC_COMMENT,
 };
 
 void *tree_sitter_rust_external_scanner_create() { return NULL; }
@@ -143,7 +145,63 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
 
   if (lexer->lookahead == '/') {
     advance(lexer);
+
+    if ((valid_symbols[LINE_COMMENT] || valid_symbols[DOC_COMMENT]) && lexer->lookahead == '/') {
+      advance(lexer);
+
+      if (lexer->lookahead == '/') {
+        advance(lexer);
+
+        // If a fourth slash is found, the line turns back to a normal comment
+        if (lexer->lookahead != '/') {
+          lexer->result_symbol = DOC_COMMENT;
+          while (true) {
+            while (lexer->lookahead != '\n') {
+              advance(lexer);
+            }
+            if (lexer->lookahead == 0) {
+              break;
+            }
+
+            lexer->mark_end(lexer);
+            advance(lexer);
+            if (lexer->lookahead == '/') {
+              advance(lexer);
+              if (lexer->lookahead == '/') {
+                advance(lexer);
+                if (lexer->lookahead == '/') {
+                  advance(lexer);
+                  // If a fourth slash is found, the line turns back to a normal comment
+                  if (lexer->lookahead == '/') {
+                    break;
+                  }
+                } else {
+                  break;
+                }
+              } else {
+                break;
+              }
+            } else {
+              break;
+            }
+          }
+        }
+      }
+
+      // Might have broken from the loop above having already processed a doc
+      // comment
+      if (lexer->result_symbol != DOC_COMMENT) {
+        lexer->result_symbol = LINE_COMMENT;
+        while (lexer->lookahead != '\n' && lexer->lookahead != 0) {
+          advance(lexer);
+        }
+      }
+
+      return true;
+    }
+
     if (lexer->lookahead != '*') return false;
+
     advance(lexer);
 
     bool after_star = false;