Refactor Rust libraries (#5654)

Follow up for #5645 and  https://github.com/edgedb/edgedb-cli/pull/1080.

- removes "python" from default features of edgeql-parser,
- renames edgeql-rust to edgeql-parser-python,
- moves edgeql-parser-* libraries under edgeql-parser dir,

Cargo.lock

@@ -227,7 +227,7 @@ dependencies = [
 ]
 
 [[package]]
-name = "edgeql-rust"
+name = "edgeql-parser-python"
 version = "0.1.0"
 dependencies = [
  "bigdecimal",

Cargo.toml

@@ -1,8 +1,8 @@
 [workspace]
 members = [
     "edb/edgeql-parser",
-    "edb/edgeql-parser-derive",
-    "edb/edgeql-rust",
+    "edb/edgeql-parser/edgeql-parser-derive",
+    "edb/edgeql-parser/edgeql-parser-python",
     "edb/graphql-rewrite",
 ]
 

MANIFEST.in

@@ -3,4 +3,4 @@ recursive-include tests *.edgeql *.esdl *.py
 include LICENSE README.md logo.svg
 
 recursive-include edb/edgeql-parser *
-recursive-include edb/edgeql-rust *
+recursive-include edb/edgeql-parser/edgeql-parser-python *

edb/common/context.py

@@ -36,7 +36,7 @@ from __future__ import annotations
 import re
 import bisect
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 from edb.common import ast
 from edb.common import markup
@@ -68,7 +68,7 @@ class ParserContext(markup.MarkupExceptionContext):
         return dic
 
     def _calc_points(self):
-        self._points = _edgeql_rust.SourcePoint.from_offsets(
+        self._points = _edgeql_parser.SourcePoint.from_offsets(
             self.buffer.encode('utf-8'),
             [self.start, self.end]
         )

edb/common/parsing.py

@@ -31,7 +31,7 @@ import parsing
 
 from edb.common.exceptions import add_context, get_context
 from edb.common import context as pctx
-from edb._edgeql_rust import TokenizerError
+from edb._edgeql_parser import TokenizerError
 from edb.errors import EdgeQLSyntaxError
 
 ParserContext = pctx.ParserContext

edb/edgeql-parser/Cargo.toml

@@ -18,12 +18,12 @@ wasm-bindgen = { version = "0.2", features = [
 serde = { version = "1.0.106", features = ["derive"], optional = true }
 thiserror = "1.0.23"
 unicode-width = "0.1.8"
-edgeql-parser-derive = { path = "../edgeql-parser-derive", optional = true }
+edgeql-parser-derive = { path = "edgeql-parser-derive", optional = true }
 cpython = { version = "0.7.0", optional = true }
 indexmap = "1.9.3"
 
 [features]
-default = ["python"]
+default = []
 wasm-lexer = ["wasm-bindgen", "serde"]
 python = ["cpython", "edgeql-parser-derive"]
 

edb/edgeql-parser/edgeql-parser-python/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "edgeql-rust"
+name = "edgeql-parser-python"
 license = "MIT/Apache-2.0"
 version = "0.1.0"
 authors = ["MagicStack Inc. <hello@magic.io>"]
@@ -7,7 +7,7 @@ rust-version = "1.59"
 edition = "2021"
 
 [dependencies]
-edgeql-parser = {path = "../edgeql-parser"}
+edgeql-parser = {path = ".."}
 bytes = "1.0.1"
 num-bigint = "0.4.3"
 bigdecimal = "0.3.0"

edb/edgeql-parser/edgeql-parser-python/src/errors.rs

@@ -54,7 +54,7 @@ impl cpython::PythonObjectWithTypeObject for TokenizerError {
             if TYPE_OBJECT.is_null() {
                 TYPE_OBJECT = PyErr::new_type(
                     py,
-                    "edb._edgeql_rust.TokenizerError",
+                    "edb._edgeql_parser.TokenizerError",
                     Some(PythonObject::into_object(py.get_type::<Exception>())),
                     None).as_type_ptr();
             }

edb/edgeql-parser/edgeql-parser-python/src/lib.rs

@@ -17,9 +17,9 @@ use pynormalize::normalize;
 use tokenizer::{get_unpickle_fn, tokenize, Token};
 
 py_module_initializer!(
-    _edgeql_rust,
-    init_edgeql_rust,
-    PyInit__edgeql_rust,
+    _edgeql_parser,
+    init_edgeql_parser,
+    PyInit__edgeql_parser,
     |py, m| {
         tokenizer::init_module(py);
         let keywords = keywords::get_keywords(py)?;

edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs

@@ -77,7 +77,7 @@ pub fn serialize_extra(variables: &[Variable]) -> Result<Bytes, String> {
         match var.value {
             Value::Int(v) => {
                 codec::Int64
-                    .encode(&mut buf, &P::Int64(v as i64))
+                    .encode(&mut buf, &P::Int64(v))
                     .map_err(|e| format!("int cannot be encoded: {}", e))?;
             }
             Value::String(ref v) => {

edb/edgeql-parser/src/tokenizer.rs

@@ -1031,3 +1031,9 @@ impl <'a> std::cmp::PartialEq for Token<'a> {
         self.kind == other.kind && self.text == other.text && self.value == other.value
     }
 }
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.message)
+    }
+}

edb/edgeql-parser/src/validation.rs

@@ -84,7 +84,7 @@ impl<'a> Validator<'a> {
         }
 
         self.keyword_buf.clear();
-        self.keyword_buf.push_str(&text);
+        self.keyword_buf.push_str(text);
         self.keyword_buf.make_ascii_lowercase();
         match &self.keyword_buf[..] {
             "named" => {
@@ -192,10 +192,10 @@ pub fn parse_value(token: &Token<'_>) -> Result<Option<Value>, String> {
             )));
         }
         BinStr => {
-            return unquote_bytes(&text).map(Value::Bytes).map(Some);
+            return unquote_bytes(text).map(Value::Bytes).map(Some);
         }
 
-        Str => unquote_string(&text)
+        Str => unquote_string(text)
             .map_err(|s| s.to_string())?
             .to_string(),
         BacktickName => text[1..text.len() - 1].replace("``", "`"),

edb/edgeql/hasher.py

@@ -19,4 +19,4 @@
 from __future__ import annotations
 from typing import *
 
-from edb._edgeql_rust import Hasher  # noqa
+from edb._edgeql_parser import Hasher  # noqa

edb/edgeql/parser/grammar/keywords.py

@@ -22,25 +22,25 @@ from __future__ import annotations
 import re
 from typing import *
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 
 keyword_types = range(1, 5)
 (UNRESERVED_KEYWORD, RESERVED_KEYWORD, TYPE_FUNC_NAME_KEYWORD,
  PARTIAL_RESERVED_KEYWORD) = keyword_types
 
-unreserved_keywords = _edgeql_rust.unreserved_keywords
-future_reserved_keywords = _edgeql_rust.future_reserved_keywords
+unreserved_keywords = _edgeql_parser.unreserved_keywords
+future_reserved_keywords = _edgeql_parser.future_reserved_keywords
 reserved_keywords = (
     future_reserved_keywords |
-    _edgeql_rust.current_reserved_keywords
+    _edgeql_parser.current_reserved_keywords
 )
 # These keywords can be used in pretty much all the places where they are
 # preceeded by a reserved keyword or some other disambiguating token like `.`,
 # `.<`, or `@`.
 #
 # In practice we mainly relax their usage as link/property names.
-partial_reserved_keywords = _edgeql_rust.partial_reserved_keywords
+partial_reserved_keywords = _edgeql_parser.partial_reserved_keywords
 
 
 def _check_keywords():

edb/edgeql/parser/grammar/rust_lexer.py

@@ -23,7 +23,7 @@ from typing import *
 from collections import deque
 
 from edb.edgeql import tokenizer
-from edb._edgeql_rust import Token
+from edb._edgeql_parser import Token
 
 
 class EdgeQLLexer(object):

edb/edgeql/tokenizer.py

@@ -22,8 +22,8 @@ from typing import *
 import re
 import hashlib
 
-from edb._edgeql_rust import tokenize as _tokenize, TokenizerError, Token
-from edb._edgeql_rust import normalize as _normalize, Entry
+from edb._edgeql_parser import tokenize as _tokenize, TokenizerError, Token
+from edb._edgeql_parser import normalize as _normalize, Entry
 
 from edb import errors
 

edb/pgsql/metaschema.py

@@ -25,7 +25,7 @@ from typing import *
 import re
 import textwrap
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 from edb.common import context as parser_context
 from edb.common import debug
@@ -7394,7 +7394,7 @@ async def execute_sql_script(
             text = e.get_field('q')
 
         elif pl_func_line:
-            point = _edgeql_rust.offset_of_line(sql_text, pl_func_line)
+            point = _edgeql_parser.offset_of_line(sql_text, pl_func_line)
             text = sql_text
 
         if point is not None:

setup.py

@@ -546,11 +546,10 @@ class ci_helper(setuptools.Command):
         elif self.type == 'rust':
             rust_hash = hash_dirs([
                 (pkg_dir / 'edgeql-parser', '.rs'),
-                (pkg_dir / 'edgeql-rust', '.rs'),
                 (pkg_dir / 'graphql-rewrite', '.rs'),
             ], extra_files=[
                 pkg_dir / 'edgeql-parser/Cargo.toml',
-                pkg_dir / 'edgeql-rust/Cargo.toml',
+                pkg_dir / 'edgeql-parser/edgeql-parser-python/Cargo.toml',
                 pkg_dir / 'graphql-rewrite/Cargo.toml',
             ])
             print(binascii.hexlify(rust_hash).decode())
@@ -1101,8 +1100,8 @@ setuptools.setup(
     ],
     rust_extensions=[
         setuptools_rust.RustExtension(
-            "edb._edgeql_rust",
-            path="edb/edgeql-rust/Cargo.toml",
+            "edb._edgeql_parser",
+            path="edb/edgeql-parser/edgeql-parser-python/Cargo.toml",
             binding=setuptools_rust.Binding.RustCPython,
         ),
         setuptools_rust.RustExtension(