[WIP] Unicode lexer

packages/css-property-parser/lib/Standard.re

@@ -276,6 +276,23 @@ let flex_value =
     | _ => Error(["expected flex_value"]),
   );
 
+let urange =
+  token(
+    fun
+    | UNICODE(range) => {
+        Ok(`Single(range));
+      }
+    | UNICODE_WILDCARD(hex, wildcard) => {
+        Ok(`Wildcard(hex, wildcard));
+      }
+    | UNICODE_RANGE(start, end_) => {
+        Ok(`Range(start, end_));
+      }
+    | _ => {
+        Error(["expected urange"]);
+      },
+  );
+
 // TODO: workarounds
 let invalid = expect(STRING("not-implemented"));
 let attr_name = invalid;
@@ -298,6 +315,5 @@ let declaration = invalid;
 let y = invalid;
 let x = invalid;
 let decibel = invalid;
-let urange = invalid;
 let semitones = invalid;
 let url_token = invalid;

packages/css-property-parser/lib/Standard.rei

@@ -80,6 +80,8 @@ let line_names: Rule.rule((unit, list(string), unit));
 
 let flex_value: Rule.rule([> | `Fr(float)]);
 
+let urange: Rule.rule([> | `Single(string) | `Range(string, string) | `Wildcard(string, string)]);
+
 let invalid: Rule.rule(unit);
 
 let attr_name: Rule.rule(unit);
@@ -122,8 +124,6 @@ let x: Rule.rule(unit);
 
 let decibel: Rule.rule(unit);
 
-let urange: Rule.rule(unit);
-
 let semitones: Rule.rule(unit);
 
 let url_token: Rule.rule(unit);

packages/css-property-parser/ppx/Generate.re

@@ -313,6 +313,7 @@ module Make = (Builder: Ppxlib.Ast_builder.S) => {
       type_("interpolation", [%type: list(string)]),
       type_("flex_value", [%type: [ | `Fr(float)]]),
       type_("line_names", [%type: (unit, list(string), unit)]),
+      type_("urange", [%type: [ | `Single(string) | `Range(string, string) | `Wildcard(string, string)]]),
       // From Parser_helper, those are `invalid` represented here as unit
       type_("ident_token", [%type: unit]),
       type_("function_token", [%type: unit]),
@@ -323,7 +324,6 @@ module Make = (Builder: Ppxlib.Ast_builder.S) => {
       type_("declaration_value", [%type: unit]),
       type_("zero", [%type: unit]),
       type_("decibel", [%type: unit]),
-      type_("urange", [%type: unit]),
       type_("semitones", [%type: unit]),
       type_("an_plus_b", [%type: unit]),
     ];

packages/css-property-parser/test/snapshots/Spec.expected.re

@@ -78,6 +78,11 @@ module Types = {
   and interpolation = list(string)
   and flex_value = [ | `Fr(float)]
   and line_names = (unit, list(string), unit)
+  and urange = [
+    | `Single(string)
+    | `Range(string, string)
+    | `Wildcard(string, string)
+  ]
   and ident_token = unit
   and function_token = unit
   and string_token = unit
@@ -87,7 +92,6 @@ module Types = {
   and declaration_value = unit
   and zero = unit
   and decibel = unit
-  and urange = unit
   and semitones = unit
   and an_plus_b = unit;
 };

packages/parser/lib/Lexer.re

@@ -33,10 +33,11 @@ let hex_digit = [%sedlex.regexp? digit | 'A' .. 'F' | 'a' .. 'f'];
 
 let up_to_6_hex_digits = [%sedlex.regexp? Rep(hex_digit, 1 .. 6)];
 
-let unicode = [%sedlex.regexp? ('\\', up_to_6_hex_digits, Opt(whitespace))];
+let unicode = [%sedlex.regexp? (up_to_6_hex_digits, Opt(whitespace))];
+
+let unicode_wildcard = [%sedlex.regexp? Rep(hex_digit | '?', 1 .. 6)];
 
 let unicode_range = [%sedlex.regexp?
-  Rep(hex_digit | '?', 1 .. 6) |
   (up_to_6_hex_digits, '-', up_to_6_hex_digits)
 ];
 
@@ -816,6 +817,24 @@ let consume_numeric = lexbuf => {
   };
 };
 
+let extract_unicode_parts = lexbuf => {
+  let unicode_str =
+    String.sub(lexeme(lexbuf), 2, String.length(lexeme(lexbuf)) - 2);
+
+  let (hex, wildcard) = String.fold_left(
+    ((hex_acc, wildcard_acc), char) => 
+      switch (char) {
+      | '0'..'9' | 'A'..'F' | 'a'..'f' => (hex_acc ++ String.make(1, char), wildcard_acc)
+      | '?' => (hex_acc, wildcard_acc ++ "?")
+      | _ => (hex_acc, wildcard_acc) // Ignore other characters
+      },
+    ("", ""),
+    unicode_str
+  );
+
+  Ok(Tokens.UNICODE_WILDCARD(hex, wildcard));
+};
+
 let consume = lexbuf => {
   let consume_hash = () =>
     switch%sedlex (lexbuf) {
@@ -849,6 +868,21 @@ let consume = lexbuf => {
     };
   switch%sedlex (lexbuf) {
   | whitespace => Ok(consume_whitespace_(lexbuf))
+  | (_u, '+', unicode) =>
+    Ok(
+      UNICODE(
+        String.sub(lexeme(lexbuf), 2, String.length(lexeme(lexbuf)) - 2),
+      ),
+    )
+  | (_u, '+', unicode_range) =>
+    switch (
+      String.sub(lexeme(lexbuf), 2, String.length(lexeme(lexbuf)) - 2)
+      |> String.split_on_char('-')
+    ) {
+    | [start, end_] => Ok(UNICODE_RANGE(start, end_))
+    | _ => unreachable(lexbuf)
+    }
+  | (_u, '+', unicode_wildcard) => extract_unicode_parts(lexbuf)
   | "\"" => consume_string("\"", lexbuf)
   | "#" => consume_hash()
   | "'" => consume_string("'", lexbuf)

packages/parser/lib/Tokens.re

@@ -22,7 +22,10 @@ type token =
   | LEFT_PAREN // <(-token>
   | RIGHT_PAREN // <)-token>
   | LEFT_BRACE // <{-token>
-  | RIGHT_BRACE; // <}-token>
+  | RIGHT_BRACE // <}-token>
+  | UNICODE(string)
+  | UNICODE_WILDCARD(string, string)
+  | UNICODE_RANGE(string, string);
 
 let string_of_char = c => String.make(1, c);
 
@@ -62,7 +65,10 @@ let humanize =
   | LEFT_PAREN => "("
   | RIGHT_PAREN => ")"
   | LEFT_BRACE => "{"
-  | RIGHT_BRACE => "}";
+  | RIGHT_BRACE => "}"
+  | UNICODE(s) => "unicode" ++ s
+  | UNICODE_WILDCARD(s, w) => "unicode-wildcard: " ++ s ++ w
+  | UNICODE_RANGE(s, e) => "unicode-range:" ++ s ++ "-" ++ e;
 
 /* TODO: This should render Token, not Parser.token */
 let token_to_string =

packages/ppx/src/Property_to_runtime.re

@@ -4905,6 +4905,22 @@ let text_orientation =
 
 let touch_action = unsupportedProperty(Property_parser.property_touch_action);
 
+let unicode_range =
+  monomorphic(
+    Property_parser.property_unicode_range,
+    (~loc) => [%expr CSS.unicode_range],
+    (~loc, value) =>
+      Builder.pexp_array(
+        ~loc,
+        List.map(
+        fun
+        | `Single(v) => [%expr `single([%e render_string(~loc, v)])]
+        | `Range(v1, v2) => [%expr `range([%e render_string(~loc, v1)], [%e render_string(~loc, v2)])]
+        | `Wildcard(v1, v2) => [%expr `wildcard([%e render_string(~loc, v1)], [%e render_string(~loc, v2)])]
+        , value),
+      ),
+  );
+
 let user_select =
   monomorphic(
     Property_parser.property_user_select,
@@ -5311,6 +5327,7 @@ let properties = [
   ("transition-timing-function", found(transition_timing_function)),
   ("transition", found(transition)),
   ("translate", found(translate)),
+  ("unicode-range", found(unicode_range)),
   ("user-select", found(user_select)),
   ("vertical-align", found(vertical_align)),
   ("visibility", found(visibility)),

packages/ppx/test/css-support/fonts-module.t/input.re

@@ -130,3 +130,7 @@
 [%css {|font-feature-settings: 'smcp' on|}];
 [%css {|font-feature-settings: 'liga' off|}];
 [%css {|font-feature-settings: 'smcp', 'swsh' 2|}];
+[%css {|unicode-range: U+3FF|}];
+[%css {|unicode-range: U+4??|}];
+[%css {|unicode-range: U+0025-00FF|}];
+[%css {|unicode-range: U+0025-00FF, U+0025|}];

packages/ppx/test/css-support/fonts-module.t/run.t

@@ -155,3 +155,10 @@ If this test fail means that the module is not in sync with the ppx
   CSS.unsafe({js|fontFeatureSettings|js}, {js|'smcp' on|js});
   CSS.unsafe({js|fontFeatureSettings|js}, {js|'liga' off|js});
   CSS.unsafe({js|fontFeatureSettings|js}, {js|'smcp', 'swsh' 2|js});
+  CSS.unicode_range([|`single({js|3FF|js})|]);
+  CSS.unicode_range([|`wildcard(({js|4|js}, {js|??|js}))|]);
+  CSS.unicode_range([|`range(({js|0025|js}, {js|00FF|js}))|]);
+  CSS.unicode_range([|
+    `range(({js|0025|js}, {js|00FF|js})),
+    `single({js|0025|js}),
+  |]);