Implement CEL optional conformance support

conformance/src/test/java/org/projectnessie/cel/conformance/SimpleConformanceTest.java

@@ -135,6 +135,7 @@ class SimpleConformanceTest {
           "math_ext.textproto",
           "namespace.textproto",
           "network_ext.textproto",
+          "optionals.textproto",
           "parse.textproto",
           "plumbing.textproto",
           "proto2.textproto",
@@ -185,12 +186,7 @@ class SimpleConformanceTest {
           "enums/strong_proto3/convert_int_too_big",
           "enums/strong_proto3/convert_int_too_neg",
           "enums/strong_proto3/convert_string",
-          "enums/strong_proto3/convert_string_bad",
-          // Optional list/map/message syntax and runtime support is not implemented yet.
-          "block_ext/basic/optional_list",
-          "block_ext/basic/optional_map",
-          "block_ext/basic/optional_map_chained",
-          "block_ext/basic/optional_message");
+          "enums/strong_proto3/convert_string_bad");
 
   private static final Set<String> matchedSkips = new LinkedHashSet<>();
   private static final AtomicInteger total = new AtomicInteger();
@@ -353,6 +349,9 @@ private static ParsedExpr parse(SimpleTest test) {
       if (usesTestOnlyBlockMacros(test.getExpr())) {
         parseOptions.add(macros(Macro.TestOnlyBlockMacros));
       }
+      if (usesOptionals(test.getExpr())) {
+        parseOptions.add(optionals());
+      }
 
       Env env = newEnv(parseOptions.toArray(new EnvOption[0]));
       AstIssuesTuple astIss = env.parse(sourceText);
@@ -439,7 +438,7 @@ private static List<EnvOption> conformanceEnvOptions(SimpleTest test, EnvOption.
       if (usesNetworkExtensions(test.getExpr())) {
         envOptions.add(network());
       }
-      if (test.getExpr().contains("optional.")) {
+      if (usesOptionals(test.getExpr())) {
         envOptions.add(optionals());
       }
       envOptions.addAll(List.of(options));
@@ -462,6 +461,13 @@ private static boolean usesStringExtensions(String expression) {
           || expression.contains(".reverse(");
     }
 
+    private static boolean usesOptionals(String expression) {
+      return expression.contains("optional.")
+          || expression.contains(".?")
+          || expression.contains("[?")
+          || expression.contains("{?");
+    }
+
     private static boolean usesNetworkExtensions(String expression) {
       return expression.contains("ip(")
           || expression.contains("cidr(")

core/src/main/congocc/cel/cel.ccc

@@ -50,16 +50,16 @@ Unary :
 Member :
   Primary
   (
-    <DOT> Field [<LPAREN> (<RPAREN> | ExprList <RPAREN>)]
-  | <LBRACKET> Expr <RBRACKET>
+    <DOT> [<QUESTIONMARK>] Field [<LPAREN> (<RPAREN> | ExprList <RPAREN>)]
+  | <LBRACKET> [<QUESTIONMARK>] Expr <RBRACKET>
   | <LBRACE> [FieldInitializerList] [<COMMA>] <RBRACE>
   )*!
   ;
 
 Primary :
     [<DOT>] <IDENTIFIER> [<LPAREN> (<RPAREN> | ExprList <RPAREN>)]
   | <LPAREN> Expr <RPAREN>
-  | <LBRACKET> (<RBRACKET> | ExprList [<COMMA>] <RBRACKET>)
+  | <LBRACKET> (<RBRACKET> | ListInitializerList [<COMMA>] <RBRACKET>)
   | <LBRACE> (<RBRACE> | MapInitializerList [<COMMA>] <RBRACE>)
   | ConstantLiteral
   ;
@@ -68,8 +68,12 @@ ExprList :
   Expr (<COMMA> Expr =>||)*!
   ;
 
+ListInitializerList :
+  [<QUESTIONMARK>] Expr (<COMMA> [<QUESTIONMARK>] Expr =>||)*!
+  ;
+
 FieldInitializerList :
-  Field <COLON> Expr (<COMMA> Field <COLON> Expr =>||)*!
+  [<QUESTIONMARK>] Field <COLON> Expr (<COMMA> [<QUESTIONMARK>] Field <COLON> Expr =>||)*!
   ;
 
 Field :
@@ -78,7 +82,7 @@ Field :
   ;
 
 MapInitializerList :
-  Expr <COLON> Expr (<COMMA> Expr <COLON> Expr =>||)*!
+  [<QUESTIONMARK>] Expr <COLON> Expr (<COMMA> [<QUESTIONMARK>] Expr <COLON> Expr =>||)*!
   ;
 
 ConstantLiteral :

core/src/main/java/org/projectnessie/cel/checker/Checker.java

@@ -56,6 +56,7 @@
 import org.projectnessie.cel.common.Location;
 import org.projectnessie.cel.common.Source;
 import org.projectnessie.cel.common.containers.Container;
+import org.projectnessie.cel.common.operators.Operator;
 import org.projectnessie.cel.common.types.Err.ErrException;
 import org.projectnessie.cel.common.types.ref.FieldType;
 import org.projectnessie.cel.parser.Parser.ParseResult;
@@ -264,10 +265,23 @@ void checkSelect(Expr.Builder e) {
       }
     }
 
+    Type resultType = checkSelectField(e, sel.getOperandBuilder(), sel.getField(), false);
+    if (sel.getTestOnly()) {
+      resultType = Decls.Bool;
+    }
+    setType(e, resultType);
+  }
+
+  private Type checkSelectField(
+      Expr.Builder e, Expr.Builder operand, String field, boolean optionalSelect) {
     // Interpret as field selection, first traversing down the operand.
-    check(sel.getOperandBuilder());
+    check(operand);
 
-    Type targetType = getType(sel.getOperandBuilder());
+    Type targetType = getType(operand);
+    boolean optionalOperand = isOptionalType(targetType);
+    if (optionalOperand) {
+      targetType = optionalValueType(targetType);
+    }
     // Assume error type by default as most types do not support field selection.
     Type resultType = Decls.Error;
     switch (kindOf(targetType)) {
@@ -279,12 +293,14 @@ void checkSelect(Expr.Builder e) {
       case kindObject:
         // Objects yield their field type declaration as the selection result type, but only if
         // the field is defined.
-        FieldType fieldType =
-            lookupFieldType(location(e), targetType.getMessageType(), sel.getField());
+        FieldType fieldType = lookupFieldType(location(e), targetType.getMessageType(), field);
         if (fieldType != null) {
           resultType = fieldType.type;
         }
         break;
+      case kindAbstract:
+        errors.typeDoesNotSupportFieldSelection(location(e), targetType);
+        break;
       case kindTypeParam:
         // Set the operand type to DYN to prevent assignment to a potentionally incorrect type
         // at a later point in type-checking. The isAssignable call will update the type
@@ -303,10 +319,16 @@ void checkSelect(Expr.Builder e) {
         }
         break;
     }
-    if (sel.getTestOnly()) {
-      resultType = Decls.Bool;
+    if (optionalOperand || optionalSelect) {
+      return Decls.newAbstractType("optional_type", Collections.singletonList(resultType));
     }
-    setType(e, resultType);
+    return resultType;
+  }
+
+  private static boolean isOptionalType(Type type) {
+    return type != null
+        && type.hasAbstractType()
+        && "optional_type".equals(type.getAbstractType().getName());
   }
 
   private boolean isQualifiedLocalVariableSelection(Expr.Builder e) {
@@ -327,6 +349,20 @@ void checkCall(Expr.Builder e) {
     List<Expr.Builder> args = call.getArgsBuilderList();
     String fnName = call.getFunction();
 
+    if (fnName.equals(Operator.OptionalSelect.id)) {
+      Decl fn = env.lookupFunction(fnName);
+      if (fn == null) {
+        for (Expr.Builder arg : args) {
+          check(arg);
+        }
+        errors.undeclaredReference(location(e), env.container.name(), fnName);
+        setType(e, Decls.Error);
+        return;
+      }
+      checkOptionalSelect(e, call, args);
+      return;
+    }
+
     // Traverse arguments.
     for (Expr.Builder arg : args) {
       check(arg);
@@ -381,6 +417,27 @@ void checkCall(Expr.Builder e) {
     errors.undeclaredReference(location(e), env.container.name(), fnName);
   }
 
+  private void checkOptionalSelect(Expr.Builder e, Call.Builder call, List<Expr.Builder> args) {
+    if (call.getTarget() != Expr.getDefaultInstance() || args.size() != 2) {
+      errors.noMatchingOverload(location(e), call.getFunction(), List.of(), false);
+      setType(e, Decls.Error);
+      return;
+    }
+
+    Expr.Builder field = args.get(1);
+    check(field);
+    if (field.getExprKindCase() != Expr.ExprKindCase.CONST_EXPR
+        || field.getConstExpr().getConstantKindCase() != Constant.ConstantKindCase.STRING_VALUE) {
+      errors.typeMismatch(location(field), Decls.String, getType(field));
+      setType(e, Decls.Error);
+      return;
+    }
+
+    Type resultType = checkSelectField(e, args.get(0), field.getConstExpr().getStringValue(), true);
+    setType(e, resultType);
+    setReference(e, newFunctionReference(Collections.singletonList("optional_select")));
+  }
+
   void resolveOverloadOrError(
       Location loc, Expr.Builder e, Decl fn, Expr.Builder target, List<Expr.Builder> args) {
     // Attempt to resolve the overload.
@@ -469,10 +526,18 @@ OverloadResolution resolveOverload(
   void checkCreateList(Expr.Builder e) {
     CreateList.Builder create = e.getListExprBuilder();
     Type elemType = null;
+    boolean[] optionalIndices = new boolean[create.getElementsCount()];
+    for (int index : create.getOptionalIndicesList()) {
+      optionalIndices[index] = true;
+    }
     for (int i = 0; i < create.getElementsBuilderList().size(); i++) {
       Expr.Builder el = create.getElementsBuilderList().get(i);
       check(el);
-      elemType = joinTypes(location(el), elemType, getType(el));
+      Type type = getType(el);
+      if (optionalIndices[i]) {
+        type = unwrapOptionalEntry(location(el), type);
+      }
+      elemType = joinTypes(location(el), elemType, type);
     }
     if (elemType == null) {
       // If the list is empty, assign free type var to elem type.
@@ -501,7 +566,11 @@ void checkCreateMap(Expr.Builder e) {
 
       Expr.Builder val = ent.getValueBuilder();
       check(val);
-      valueType = joinTypes(location(val), valueType, getType(val));
+      Type type = getType(val);
+      if (ent.getOptionalEntry()) {
+        type = unwrapOptionalEntry(location(val), type);
+      }
+      valueType = joinTypes(location(val), valueType, type);
     }
     if (keyType == null) {
       // If the map is empty, assign free type variables to typeKey and value type.
@@ -553,12 +622,33 @@ void checkCreateMessage(Expr.Builder e) {
       if (t != null) {
         fieldType = t.type;
       }
-      if (!isAssignable(fieldType, getType(value))) {
+      Type valueType = getType(value);
+      if (ent.getOptionalEntry()) {
+        valueType = unwrapOptionalEntry(location(value), valueType);
+      }
+      if (!isAssignable(fieldType, valueType)) {
         errors.fieldTypeMismatch(locationByID(ent.getId()), field, fieldType, getType(value));
       }
     }
   }
 
+  private static Type optionalValueType(Type type) {
+    if (!isOptionalType(type) || type.getAbstractType().getParameterTypesCount() == 0) {
+      return null;
+    }
+    return type.getAbstractType().getParameterTypes(0);
+  }
+
+  private Type unwrapOptionalEntry(Location location, Type type) {
+    Type unwrapped = optionalValueType(type);
+    if (unwrapped != null || isDyn(type)) {
+      return unwrapped != null ? unwrapped : type;
+    }
+    errors.typeMismatch(
+        location, Decls.newAbstractType("optional_type", Collections.singletonList(type)), type);
+    return type;
+  }
+
   void checkComprehension(Expr.Builder e) {
     Comprehension.Builder comp = e.getComprehensionExprBuilder();
     check(comp.getIterRangeBuilder());

core/src/main/java/org/projectnessie/cel/common/operators/Operator.java

@@ -42,6 +42,8 @@ public enum Operator {
   Modulo("_%_", 3, "%"),
   Negate("-_", 2, "-"),
   Index("_[_]", 1, null),
+  OptionalSelect("@optional_select"),
+  OptionalIndex("@optional_index"),
   // Macros, must have a valid identifier.
   Has("has"),
   All("all"),