From 43eec3279662b9ccc9b331db91ac221c69f3acab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?=
 <aljazerzen@users.noreply.github.com>
Date: Wed, 14 Jun 2023 18:39:50 +0200
Subject: [PATCH] Refactor Rust libraries (#5654)

Follow up for #5645 and  https://github.com/edgedb/edgedb-cli/pull/1080.

- removes "python" from default features of edgeql-parser,
- renames edgeql-rust to edgeql-parser-python,
- moves edgeql-parser-* libraries under edgeql-parser dir,
---
 Cargo.lock                                             |  2 +-
 Cargo.toml                                             |  4 ++--
 MANIFEST.in                                            |  2 +-
 edb/common/context.py                                  |  4 ++--
 edb/common/parsing.py                                  |  2 +-
 edb/edgeql-parser/Cargo.toml                           |  4 ++--
 .../edgeql-parser-derive/Cargo.toml                    |  0
 .../edgeql-parser-derive/src/lib.rs                    |  0
 .../edgeql-parser-python}/Cargo.toml                   |  4 ++--
 .../edgeql-parser-python}/src/errors.rs                |  2 +-
 .../edgeql-parser-python}/src/hash.rs                  |  0
 .../edgeql-parser-python}/src/keywords.rs              |  0
 .../edgeql-parser-python}/src/lib.rs                   |  6 +++---
 .../edgeql-parser-python}/src/normalize.rs             |  0
 .../edgeql-parser-python}/src/position.rs              |  0
 .../edgeql-parser-python}/src/pynormalize.rs           |  2 +-
 .../edgeql-parser-python}/src/tokenizer.rs             |  0
 .../edgeql-parser-python}/tests/normalize.rs           |  0
 edb/edgeql-parser/src/tokenizer.rs                     |  6 ++++++
 edb/edgeql-parser/src/validation.rs                    |  6 +++---
 edb/edgeql/hasher.py                                   |  2 +-
 edb/edgeql/parser/grammar/keywords.py                  | 10 +++++-----
 edb/edgeql/parser/grammar/rust_lexer.py                |  2 +-
 edb/edgeql/tokenizer.py                                |  4 ++--
 edb/pgsql/metaschema.py                                |  4 ++--
 setup.py                                               |  7 +++----
 26 files changed, 39 insertions(+), 34 deletions(-)
 rename edb/{ => edgeql-parser}/edgeql-parser-derive/Cargo.toml (100%)
 rename edb/{ => edgeql-parser}/edgeql-parser-derive/src/lib.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/Cargo.toml (88%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/errors.rs (97%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/hash.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/keywords.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/lib.rs (95%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/normalize.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/position.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/pynormalize.rs (99%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/src/tokenizer.rs (100%)
 rename edb/{edgeql-rust => edgeql-parser/edgeql-parser-python}/tests/normalize.rs (100%)

diff --git a/Cargo.lock b/Cargo.lock
index 2eda235b1..4df9a3a20 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -227,7 +227,7 @@ dependencies = [
 ]
 
 [[package]]
-name = "edgeql-rust"
+name = "edgeql-parser-python"
 version = "0.1.0"
 dependencies = [
  "bigdecimal",
diff --git a/Cargo.toml b/Cargo.toml
index f25eb1d3f..41e1681b9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,8 +1,8 @@
 [workspace]
 members = [
     "edb/edgeql-parser",
-    "edb/edgeql-parser-derive",
-    "edb/edgeql-rust",
+    "edb/edgeql-parser/edgeql-parser-derive",
+    "edb/edgeql-parser/edgeql-parser-python",
     "edb/graphql-rewrite",
 ]
 
diff --git a/MANIFEST.in b/MANIFEST.in
index 61fae075c..f4454a73c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,4 +3,4 @@ recursive-include tests *.edgeql *.esdl *.py
 include LICENSE README.md logo.svg
 
 recursive-include edb/edgeql-parser *
-recursive-include edb/edgeql-rust *
+recursive-include edb/edgeql-parser/edgeql-parser-python *
diff --git a/edb/common/context.py b/edb/common/context.py
index 6f68c65bd..d8bdb1873 100644
--- a/edb/common/context.py
+++ b/edb/common/context.py
@@ -36,7 +36,7 @@ from __future__ import annotations
 import re
 import bisect
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 from edb.common import ast
 from edb.common import markup
@@ -68,7 +68,7 @@ class ParserContext(markup.MarkupExceptionContext):
         return dic
 
     def _calc_points(self):
-        self._points = _edgeql_rust.SourcePoint.from_offsets(
+        self._points = _edgeql_parser.SourcePoint.from_offsets(
             self.buffer.encode('utf-8'),
             [self.start, self.end]
         )
diff --git a/edb/common/parsing.py b/edb/common/parsing.py
index c6a08d562..5fb1683a5 100644
--- a/edb/common/parsing.py
+++ b/edb/common/parsing.py
@@ -31,7 +31,7 @@ import parsing
 
 from edb.common.exceptions import add_context, get_context
 from edb.common import context as pctx
-from edb._edgeql_rust import TokenizerError
+from edb._edgeql_parser import TokenizerError
 from edb.errors import EdgeQLSyntaxError
 
 ParserContext = pctx.ParserContext
diff --git a/edb/edgeql-parser/Cargo.toml b/edb/edgeql-parser/Cargo.toml
index 980e97e15..bf4575e82 100644
--- a/edb/edgeql-parser/Cargo.toml
+++ b/edb/edgeql-parser/Cargo.toml
@@ -18,12 +18,12 @@ wasm-bindgen = { version = "0.2", features = [
 serde = { version = "1.0.106", features = ["derive"], optional = true }
 thiserror = "1.0.23"
 unicode-width = "0.1.8"
-edgeql-parser-derive = { path = "../edgeql-parser-derive", optional = true }
+edgeql-parser-derive = { path = "edgeql-parser-derive", optional = true }
 cpython = { version = "0.7.0", optional = true }
 indexmap = "1.9.3"
 
 [features]
-default = ["python"]
+default = []
 wasm-lexer = ["wasm-bindgen", "serde"]
 python = ["cpython", "edgeql-parser-derive"]
 
diff --git a/edb/edgeql-parser-derive/Cargo.toml b/edb/edgeql-parser/edgeql-parser-derive/Cargo.toml
similarity index 100%
rename from edb/edgeql-parser-derive/Cargo.toml
rename to edb/edgeql-parser/edgeql-parser-derive/Cargo.toml
diff --git a/edb/edgeql-parser-derive/src/lib.rs b/edb/edgeql-parser/edgeql-parser-derive/src/lib.rs
similarity index 100%
rename from edb/edgeql-parser-derive/src/lib.rs
rename to edb/edgeql-parser/edgeql-parser-derive/src/lib.rs
diff --git a/edb/edgeql-rust/Cargo.toml b/edb/edgeql-parser/edgeql-parser-python/Cargo.toml
similarity index 88%
rename from edb/edgeql-rust/Cargo.toml
rename to edb/edgeql-parser/edgeql-parser-python/Cargo.toml
index 7d760150a..66223c37c 100644
--- a/edb/edgeql-rust/Cargo.toml
+++ b/edb/edgeql-parser/edgeql-parser-python/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "edgeql-rust"
+name = "edgeql-parser-python"
 license = "MIT/Apache-2.0"
 version = "0.1.0"
 authors = ["MagicStack Inc. <hello@magic.io>"]
@@ -7,7 +7,7 @@ rust-version = "1.59"
 edition = "2021"
 
 [dependencies]
-edgeql-parser = {path = "../edgeql-parser"}
+edgeql-parser = {path = ".."}
 bytes = "1.0.1"
 num-bigint = "0.4.3"
 bigdecimal = "0.3.0"
diff --git a/edb/edgeql-rust/src/errors.rs b/edb/edgeql-parser/edgeql-parser-python/src/errors.rs
similarity index 97%
rename from edb/edgeql-rust/src/errors.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/errors.rs
index 61165d719..754943121 100644
--- a/edb/edgeql-rust/src/errors.rs
+++ b/edb/edgeql-parser/edgeql-parser-python/src/errors.rs
@@ -54,7 +54,7 @@ impl cpython::PythonObjectWithTypeObject for TokenizerError {
             if TYPE_OBJECT.is_null() {
                 TYPE_OBJECT = PyErr::new_type(
                     py,
-                    "edb._edgeql_rust.TokenizerError",
+                    "edb._edgeql_parser.TokenizerError",
                     Some(PythonObject::into_object(py.get_type::<Exception>())),
                     None).as_type_ptr();
             }
diff --git a/edb/edgeql-rust/src/hash.rs b/edb/edgeql-parser/edgeql-parser-python/src/hash.rs
similarity index 100%
rename from edb/edgeql-rust/src/hash.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/hash.rs
diff --git a/edb/edgeql-rust/src/keywords.rs b/edb/edgeql-parser/edgeql-parser-python/src/keywords.rs
similarity index 100%
rename from edb/edgeql-rust/src/keywords.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/keywords.rs
diff --git a/edb/edgeql-rust/src/lib.rs b/edb/edgeql-parser/edgeql-parser-python/src/lib.rs
similarity index 95%
rename from edb/edgeql-rust/src/lib.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/lib.rs
index d35335389..d650507fd 100644
--- a/edb/edgeql-rust/src/lib.rs
+++ b/edb/edgeql-parser/edgeql-parser-python/src/lib.rs
@@ -17,9 +17,9 @@ use pynormalize::normalize;
 use tokenizer::{get_unpickle_fn, tokenize, Token};
 
 py_module_initializer!(
-    _edgeql_rust,
-    init_edgeql_rust,
-    PyInit__edgeql_rust,
+    _edgeql_parser,
+    init_edgeql_parser,
+    PyInit__edgeql_parser,
     |py, m| {
         tokenizer::init_module(py);
         let keywords = keywords::get_keywords(py)?;
diff --git a/edb/edgeql-rust/src/normalize.rs b/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs
similarity index 100%
rename from edb/edgeql-rust/src/normalize.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/normalize.rs
diff --git a/edb/edgeql-rust/src/position.rs b/edb/edgeql-parser/edgeql-parser-python/src/position.rs
similarity index 100%
rename from edb/edgeql-rust/src/position.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/position.rs
diff --git a/edb/edgeql-rust/src/pynormalize.rs b/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs
similarity index 99%
rename from edb/edgeql-rust/src/pynormalize.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs
index 41f1bcfc5..29dbbebd7 100644
--- a/edb/edgeql-rust/src/pynormalize.rs
+++ b/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs
@@ -77,7 +77,7 @@ pub fn serialize_extra(variables: &[Variable]) -> Result<Bytes, String> {
         match var.value {
             Value::Int(v) => {
                 codec::Int64
-                    .encode(&mut buf, &P::Int64(v as i64))
+                    .encode(&mut buf, &P::Int64(v))
                     .map_err(|e| format!("int cannot be encoded: {}", e))?;
             }
             Value::String(ref v) => {
diff --git a/edb/edgeql-rust/src/tokenizer.rs b/edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs
similarity index 100%
rename from edb/edgeql-rust/src/tokenizer.rs
rename to edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs
diff --git a/edb/edgeql-rust/tests/normalize.rs b/edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs
similarity index 100%
rename from edb/edgeql-rust/tests/normalize.rs
rename to edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs
diff --git a/edb/edgeql-parser/src/tokenizer.rs b/edb/edgeql-parser/src/tokenizer.rs
index 4e5ad7980..21f970ade 100644
--- a/edb/edgeql-parser/src/tokenizer.rs
+++ b/edb/edgeql-parser/src/tokenizer.rs
@@ -1031,3 +1031,9 @@ impl <'a> std::cmp::PartialEq for Token<'a> {
         self.kind == other.kind && self.text == other.text && self.value == other.value
     }
 }
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.message)
+    }
+}
diff --git a/edb/edgeql-parser/src/validation.rs b/edb/edgeql-parser/src/validation.rs
index f8d2bfb16..b5a6d1f2a 100644
--- a/edb/edgeql-parser/src/validation.rs
+++ b/edb/edgeql-parser/src/validation.rs
@@ -84,7 +84,7 @@ impl<'a> Validator<'a> {
         }
 
         self.keyword_buf.clear();
-        self.keyword_buf.push_str(&text);
+        self.keyword_buf.push_str(text);
         self.keyword_buf.make_ascii_lowercase();
         match &self.keyword_buf[..] {
             "named" => {
@@ -192,10 +192,10 @@ pub fn parse_value(token: &Token<'_>) -> Result<Option<Value>, String> {
             )));
         }
         BinStr => {
-            return unquote_bytes(&text).map(Value::Bytes).map(Some);
+            return unquote_bytes(text).map(Value::Bytes).map(Some);
         }
 
-        Str => unquote_string(&text)
+        Str => unquote_string(text)
             .map_err(|s| s.to_string())?
             .to_string(),
         BacktickName => text[1..text.len() - 1].replace("``", "`"),
diff --git a/edb/edgeql/hasher.py b/edb/edgeql/hasher.py
index 424b80433..c2eeae43d 100644
--- a/edb/edgeql/hasher.py
+++ b/edb/edgeql/hasher.py
@@ -19,4 +19,4 @@
 from __future__ import annotations
 from typing import *
 
-from edb._edgeql_rust import Hasher  # noqa
+from edb._edgeql_parser import Hasher  # noqa
diff --git a/edb/edgeql/parser/grammar/keywords.py b/edb/edgeql/parser/grammar/keywords.py
index ba9304e35..0ee26b888 100644
--- a/edb/edgeql/parser/grammar/keywords.py
+++ b/edb/edgeql/parser/grammar/keywords.py
@@ -22,25 +22,25 @@ from __future__ import annotations
 import re
 from typing import *
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 
 keyword_types = range(1, 5)
 (UNRESERVED_KEYWORD, RESERVED_KEYWORD, TYPE_FUNC_NAME_KEYWORD,
  PARTIAL_RESERVED_KEYWORD) = keyword_types
 
-unreserved_keywords = _edgeql_rust.unreserved_keywords
-future_reserved_keywords = _edgeql_rust.future_reserved_keywords
+unreserved_keywords = _edgeql_parser.unreserved_keywords
+future_reserved_keywords = _edgeql_parser.future_reserved_keywords
 reserved_keywords = (
     future_reserved_keywords |
-    _edgeql_rust.current_reserved_keywords
+    _edgeql_parser.current_reserved_keywords
 )
 # These keywords can be used in pretty much all the places where they are
 # preceeded by a reserved keyword or some other disambiguating token like `.`,
 # `.<`, or `@`.
 #
 # In practice we mainly relax their usage as link/property names.
-partial_reserved_keywords = _edgeql_rust.partial_reserved_keywords
+partial_reserved_keywords = _edgeql_parser.partial_reserved_keywords
 
 
 def _check_keywords():
diff --git a/edb/edgeql/parser/grammar/rust_lexer.py b/edb/edgeql/parser/grammar/rust_lexer.py
index 11518abda..402b490f1 100644
--- a/edb/edgeql/parser/grammar/rust_lexer.py
+++ b/edb/edgeql/parser/grammar/rust_lexer.py
@@ -23,7 +23,7 @@ from typing import *
 from collections import deque
 
 from edb.edgeql import tokenizer
-from edb._edgeql_rust import Token
+from edb._edgeql_parser import Token
 
 
 class EdgeQLLexer(object):
diff --git a/edb/edgeql/tokenizer.py b/edb/edgeql/tokenizer.py
index 0d809c524..4179a6863 100644
--- a/edb/edgeql/tokenizer.py
+++ b/edb/edgeql/tokenizer.py
@@ -22,8 +22,8 @@ from typing import *
 import re
 import hashlib
 
-from edb._edgeql_rust import tokenize as _tokenize, TokenizerError, Token
-from edb._edgeql_rust import normalize as _normalize, Entry
+from edb._edgeql_parser import tokenize as _tokenize, TokenizerError, Token
+from edb._edgeql_parser import normalize as _normalize, Entry
 
 from edb import errors
 
diff --git a/edb/pgsql/metaschema.py b/edb/pgsql/metaschema.py
index 5436c1714..4a2e44929 100644
--- a/edb/pgsql/metaschema.py
+++ b/edb/pgsql/metaschema.py
@@ -25,7 +25,7 @@ from typing import *
 import re
 import textwrap
 
-from edb import _edgeql_rust
+from edb import _edgeql_parser
 
 from edb.common import context as parser_context
 from edb.common import debug
@@ -7394,7 +7394,7 @@ async def execute_sql_script(
             text = e.get_field('q')
 
         elif pl_func_line:
-            point = _edgeql_rust.offset_of_line(sql_text, pl_func_line)
+            point = _edgeql_parser.offset_of_line(sql_text, pl_func_line)
             text = sql_text
 
         if point is not None:
diff --git a/setup.py b/setup.py
index 2f7f5d607..418d4438e 100644
--- a/setup.py
+++ b/setup.py
@@ -546,11 +546,10 @@ class ci_helper(setuptools.Command):
         elif self.type == 'rust':
             rust_hash = hash_dirs([
                 (pkg_dir / 'edgeql-parser', '.rs'),
-                (pkg_dir / 'edgeql-rust', '.rs'),
                 (pkg_dir / 'graphql-rewrite', '.rs'),
             ], extra_files=[
                 pkg_dir / 'edgeql-parser/Cargo.toml',
-                pkg_dir / 'edgeql-rust/Cargo.toml',
+                pkg_dir / 'edgeql-parser/edgeql-parser-python/Cargo.toml',
                 pkg_dir / 'graphql-rewrite/Cargo.toml',
             ])
             print(binascii.hexlify(rust_hash).decode())
@@ -1101,8 +1100,8 @@ setuptools.setup(
     ],
     rust_extensions=[
         setuptools_rust.RustExtension(
-            "edb._edgeql_rust",
-            path="edb/edgeql-rust/Cargo.toml",
+            "edb._edgeql_parser",
+            path="edb/edgeql-parser/edgeql-parser-python/Cargo.toml",
             binding=setuptools_rust.Binding.RustCPython,
         ),
         setuptools_rust.RustExtension(