Drift files: Support annotations in comments

Closes #3057

Author: simolus3

docs/content/sql_api/drift_files.md

@@ -42,6 +42,17 @@ what we got:
   above, thats the `AllTodosResult` class, which contains all fields from
   `todos` and the description of the associated category.
 
+!!! tip "Comment syntax for better editor support"
+
+    While drift files are mostly just SQL, `import` statements and names for
+    statements prevents some editors from recognizing that.
+    Starting from drift 2.32, you can also write drift files using comments:
+
+    <Snippet href="/lib/src/snippets/drift_files/tables_comments.drift" name="(full)" />
+
+    Configuring your editor to treat drift files as SQL text then allows syntax highlighting
+    and formatting tools to interpret them correctly.
+
 ## Variables
 
 Inside of named queries, you can use variables just like you would expect with

docs/lib/src/generated_snippets.dart

@@ -0,0 +1,11 @@
+-- import 'tables.drift';
+
+-- we can put named SQL queries in here as well:
+-- createEntry:
+INSERT INTO todos (title, content) VALUES (:title, :content);
+
+-- deleteById:
+DELETE FROM todos WHERE id = :id;
+
+-- allTodos:
+SELECT * FROM todos;

docs/lib/src/snippets/drift_files/tables_comments.drift

@@ -2,6 +2,7 @@
 
 - Refactor the different `parse` methods on `SqlEngine` into a single one
   taking an `ParserEntrypoint` enum.
+- In drift files, allow reading imports and statement metadata from comments.
 
 ## 0.43.1
 

sqlparser/CHANGELOG.md

@@ -6,6 +6,7 @@ import 'package:sqlparser/sqlparser.dart';
 import 'package:sqlparser/src/reader/parser.dart';
 import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
 
+import '../reader/tokenizer/token_source.dart';
 import 'autocomplete/engine.dart';
 import 'builtin_tables.dart';
 
@@ -112,10 +113,10 @@ final class SqlEngine {
     return scope;
   }
 
-  ScannerTokenSource _prepareScanning(FileSpan source) {
+  TokenSource _prepareScanning(FileSpan source) {
     final scanner =
         Scanner(source, scanDriftTokens: options.useDriftExtensions);
-    return ScannerTokenSource(scanner);
+    return TokenSource(scanner);
   }
 
   /// Tokenizes the [source] into a list list [Token]s. Each [Token] contains
@@ -129,12 +130,7 @@ final class SqlEngine {
   /// you need to filter them. When using the methods in this class, this will
   /// be taken care of automatically.
   List<Token> tokenize(FileSpan source) {
-    final scanner = _prepareScanning(source);
-    // Read until end
-    Token token;
-    do {
-      token = scanner.readToken();
-    } while (token.type != TokenType.eof);
+    final scanner = _prepareScanning(source)..readTokensUntilEnd();
 
     final errors = scanner.scanner.errors;
     if (errors.isNotEmpty) {

sqlparser/lib/src/engine/sql_engine.dart

@@ -63,7 +63,7 @@ extension Parser on ParserState {
   bool get enableDriftExtensions => options.useDriftExtensions;
 
   void _suggestHint(HintDescription description) {
-    autoComplete?.addHint(Hint(tokens.previous, description));
+    autoComplete?.addHint(Hint(tokens.lastConsumedToken, description));
   }
 
   void _suggestHintForTokens(Iterable<TokenType> types) {
@@ -81,7 +81,7 @@ extension Parser on ParserState {
   bool get _isAtEnd => _peek.type == TokenType.eof;
   Token get _peek => tokens.lookahead1();
 
-  Token get _previous => tokens.previous!;
+  Token get _previous => tokens.lastConsumedToken!;
 
   bool _match(Iterable<TokenType> types) {
     if (_reportAutoComplete) _suggestHintForTokens(types);
@@ -114,14 +114,14 @@ extension Parser on ParserState {
   /// "NOT" followed by [type]. Does not consume any tokens.
   bool _checkWithNot(TokenType type) {
     if (_check(type)) return true;
-    final (peek, next) = tokens.lookahead2();
+    final (peek, next) = tokens.keywordLookahead2();
     return peek.type == TokenType.not && next?.type == type;
   }
 
   /// Like [_checkWithNot], but with more than one token type.
   bool _checkAnyWithNot(List<TokenType> types) {
     if (types.any(_check)) return true;
-    final (peek, next) = tokens.lookahead2();
+    final (peek, next) = tokens.keywordLookahead2();
     return peek.type == TokenType.not && types.contains(next?.type);
   }
 
@@ -313,16 +313,20 @@ extension Parser on ParserState {
   }
 
   DriftFile driftFile() {
+    tokens.scanner.isInTopLevelDriftFile = true;
     final first = _peek;
-    final foundComponents = <PartOfDriftFile?>[];
+    final foundComponents = <PartOfDriftFile>[];
 
-    while (!_isAtEnd) {
-      foundComponents.add(_parseAsStatement(_partOfDriftFile));
-    }
+    while (true) {
+      tokens.scanner.isInTopLevelDriftFile = true;
+      if (_isAtEnd) break;
 
-    foundComponents.removeWhere((c) => c == null);
+      if (_parseAsStatement(_partOfDriftFile) case final component?) {
+        foundComponents.add(component);
+      }
+    }
 
-    final file = DriftFile(foundComponents.cast());
+    final file = DriftFile(foundComponents);
     if (foundComponents.isNotEmpty) {
       file.setSpan(first, _previous);
     } else {
@@ -334,6 +338,9 @@ extension Parser on ParserState {
   }
 
   PartOfDriftFile _partOfDriftFile() {
+    _peek;
+    tokens.scanner.isInTopLevelDriftFile = false;
+
     final found = _import() ?? _create() ?? _declaredStatement();
 
     if (found != null) {
@@ -3070,7 +3077,7 @@ final class _ExpressionParser extends ParserState {
 
     if (_peek is KeywordToken) {
       // Improve error messages for possible function calls, https://github.com/simolus3/drift/discussions/2277
-      final (_, next) = tokens.lookahead2();
+      final (_, next) = tokens.keywordLookahead2();
       if (next?.type == TokenType.leftParen) {
         _error(
           'Expected an expression here, but got a reserved keyword. Did you '

sqlparser/lib/src/reader/parser.dart

@@ -4,42 +4,8 @@ import 'package:charcode/charcode.dart';
 import 'package:source_span/source_span.dart';
 
 import 'token.dart';
-import 'token_source.dart';
 import 'utils.dart';
 
-final class ScannerTokenSource extends TokenSource {
-  final List<Token> tokens = [];
-  final Scanner scanner;
-
-  ScannerTokenSource(this.scanner);
-
-  int _index = 0;
-  Token? _previous;
-
-  @override
-  Token readToken() {
-    while (true) {
-      final read = scanner.scanToken();
-      if (read is TokenizerError) {
-        scanner.errors.add(read);
-        continue;
-      }
-
-      read
-        ..index = _index++
-        ..previous = _previous;
-      _previous?.next = read;
-      _previous = read;
-      tokens.add(read);
-      if (read.invisibleToParser) {
-        continue;
-      }
-
-      return read;
-    }
-  }
-}
-
 class Scanner {
   final Uint16List _charCodes;
   final List<TokenizerError> errors = [];
@@ -48,6 +14,8 @@ class Scanner {
   final bool scanDriftTokens;
   final SourceFile _file;
 
+  bool isInTopLevelDriftFile = false;
+
   /// Pending opening tokens used to associate them with closing tokens.
   ///
   /// This used to pair matching parentheses, as the information can be used by
@@ -520,13 +488,28 @@ class Scanner {
 
   /// Scans a line comment after the -- has already been read.
   CommentToken _lineComment() {
-    final contentBuilder = StringBuffer();
-    while (!_isAtEnd && _peek() != $lf) {
-      contentBuilder.writeCharCode(_nextChar());
+    var nextLineBreak = _charCodes.indexOf($lf, _currentOffset);
+    if (nextLineBreak == -1) {
+      nextLineBreak = _endOffset;
     }
 
-    return CommentToken(
-        CommentMode.line, contentBuilder.toString(), _currentSpan);
+    final content = String.fromCharCodes(
+        _charCodes.getRange(_currentOffset, nextLineBreak));
+    if (scanDriftTokens && isInTopLevelDriftFile) {
+      // We can parse line comments as import statements or named statements.
+      // We currently use fairly crude heuristics for this: The structures we
+      // want to parse end with colons or semicolons, so we'll parse those if
+      // the comment is at the right location.
+      if (_importComment.hasMatch(content) ||
+          _statementMeta.hasMatch(content)) {
+        // End the comment token without consuming the line. This will treat the
+        // initial `--` as a comment and allows us to parse the rest.
+        return CommentToken(CommentMode.line, '', _currentSpan);
+      }
+    }
+
+    _currentOffset = nextLineBreak;
+    return CommentToken(CommentMode.line, content, _currentSpan);
   }
 
   /// Scans a /* ... */ comment after the first /* has already been read.
@@ -550,4 +533,9 @@ class Scanner {
     return CommentToken(
         CommentMode.cStyle, contentBuilder.toString(), _currentSpan);
   }
+
+  static final _importComment = RegExp(r'^\s*import.*;', caseSensitive: false);
+  // match `foo:` or `myQuery (:variable AS TEXT):`
+  static final _statementMeta =
+      RegExp(r'^\s*\w+\s*(?:\(.*\)\s*)?:', caseSensitive: false);
 }

sqlparser/lib/src/reader/tokenizer/scanner.dart

@@ -1,39 +1,48 @@
 import 'package:meta/meta.dart';
 
+import 'scanner.dart';
 import 'token.dart';
 
 @internal
-abstract base class TokenSource {
-  Token? _lastConsumedToken;
-
+final class TokenSource {
+  /// A list of tokens that have been scanned.
+  ///
+  /// Note that this just collects tokens obtained through [readToken] and thus
+  /// doesn't include potential tokens of source regions that haven't been
+  /// parsed yet.
+  final List<Token> tokens = [];
+  final Scanner scanner;
+
+  int _index = 0;
+  Token? _previous;
+  Token? lastConsumedToken;
   // Up to two tokens of lookahead in used by the parser.
   Token? _pendingLookahead0;
   Token? _pendingLookahead1;
 
-  Token? get previous => _lastConsumedToken;
-
-  TokenSource();
-
-  factory TokenSource.fromIterator(Iterator<Token> tokens) {
-    return _IteratorTokenSource(tokens);
-  }
+  TokenSource(this.scanner);
 
-  Token lookahead1() {
+  Token lookahead1({bool treatCommentAsDriftAnnotation = false}) {
     if (_pendingLookahead0 case final token?) {
       return token;
     } else {
-      return _pendingLookahead0 = readToken();
+      return _pendingLookahead0 = _readToken();
     }
   }
 
-  (Token, Token?) lookahead2() {
+  /// Returns the next two tokens, but only if the next token is a keyword
+  /// token.
+  ///
+  /// This special case is used to parse `NOT IN`, `NOT BETWEEN` and similar
+  /// structures.
+  (Token, Token?) keywordLookahead2() {
     final peek = lookahead1();
-    if (peek.type == TokenType.eof) {
+    if (peek is! KeywordToken) {
       return (peek, null);
     }
 
     final next = switch (_pendingLookahead1) {
-      null => _pendingLookahead1 = readToken(),
+      null => _pendingLookahead1 = _readToken(),
       final pending => pending,
     };
     return (peek, next);
@@ -43,36 +52,40 @@ abstract base class TokenSource {
     if (_pendingLookahead0 case final token?) {
       _pendingLookahead0 = _pendingLookahead1;
       _pendingLookahead1 = null;
-      return _lastConsumedToken = token;
+      return lastConsumedToken = token;
     } else {
       // We should have rolled this into pendingLookahead0 when consuming.
       assert(_pendingLookahead1 == null);
-      return _lastConsumedToken = readToken();
+      return lastConsumedToken = _readToken();
     }
   }
 
-  @protected
-  Token readToken();
-}
-
-final class _IteratorTokenSource extends TokenSource {
-  final Iterator<Token> _tokens;
-
-  _IteratorTokenSource(this._tokens);
-
-  @override
-  Token readToken() {
+  Token _readToken() {
     while (true) {
-      if (!_tokens.moveNext()) {
-        throw StateError('Called readTokens past eof');
+      final read = scanner.scanToken();
+      if (read is TokenizerError) {
+        scanner.errors.add(read);
+        continue;
       }
 
-      final token = _tokens.current;
-      if (token.invisibleToParser) {
+      read
+        ..index = _index++
+        ..previous = _previous;
+      _previous?.next = read;
+      _previous = read;
+      tokens.add(read);
+      if (read.invisibleToParser) {
         continue;
       }
 
-      return token;
+      return read;
     }
   }
+
+  void readTokensUntilEnd() {
+    Token token;
+    do {
+      token = _readToken();
+    } while (token.type != TokenType.eof);
+  }
 }