Support "raw" identifiers

Using __X__ as identifiers for those that need to be kept as is, e.g.
not conventionalized is a mostly nice way of doing it.

One thing that still needs to consider is how to do constructors.
Since we use the convention that a constructor needs to start with
a capital, which no longer works.

One solution, e.g. new__X__, is ugly and difficult to implement. Another
would be to add e.g. a new function, to allow writing

constructor(__X__, *arguments)

This works cleanly, but is also not that nice.

tested/descriptions/converters.py

@@ -22,6 +22,7 @@
     conventionalize_property,
 )
 from tested.languages.generation import NestedTypeDeclaration, generate_type_declaration
+from tested.serialisation import Identifier
 from tested.testsuite import LanguageMapping
 from tested.utils import get_args
 
@@ -157,7 +158,7 @@ def _construct_datatype(
 
 
 def _support_language_specific_arguments(
-    normal: Callable[[Language, str], str],
+    normal: Callable[[Language, Identifier], Identifier],
     bundle: Bundle,
     actual: str | LanguageMapping,
 ) -> str:
@@ -168,7 +169,7 @@ def _support_language_specific_arguments(
         )
         return actual[bundle.config.programming_language]
     else:
-        return normal(bundle.language, actual)
+        return normal(bundle.language, Identifier(actual))
 
 
 def convert_templated_problem(bundle: Bundle, raw_description: str) -> str:

tested/dsl/ast_translator.py

@@ -13,6 +13,9 @@
 - Two special values exist: Null and Undefined.
 - Function calls whose name begins with a capital are considered constructors.
 - Identifiers in call caps are considered "global" variables.
+- Most identifiers with __ (e.g. __X__) are considered raw and will not be changed
+  to adhere to the conventions of a programming language.
+  This is currently not possible with constructors.
 
 From the Python grammar (at https://docs.python.org/3/library/ast.html#abstract-grammar),
 the following is supported:
@@ -115,13 +118,13 @@ def _convert_ann_assignment(node: ast.AnnAssign) -> Assignment:
     else:
         raise InvalidDslError("Type hints should be simple values.")
 
-    # Check if the type is built-in type or not.
+    # Check if the type is a built-in type or not.
     is_our_type = any(type_ in x.__members__.values() for x in get_args(AllTypes))
     if not is_our_type:
-        type_ = VariableType(data=type_)
+        type_ = VariableType(data=Identifier(type_))
 
     return VariableAssignment(
-        variable=node.target.id,
+        variable=Identifier(node.target.id),
         expression=value,
         type=cast(VariableType | AllTypes, type_),
     )
@@ -141,16 +144,19 @@ def _convert_assignment(node: ast.Assign) -> Assignment:
         if isinstance(value, Value):
             type_ = value.type
         elif isinstance(value, FunctionCall) and value.type == FunctionType.CONSTRUCTOR:
-            type_ = VariableType(data=value.name)
+            type_ = VariableType(data=Identifier(value.name))
 
         if not type_:
             raise InvalidDslError(
                 f"Could not deduce the type of variable {variable.id}: add a type annotation."
             )
 
         assert isinstance(type_, AllTypes | VariableType)
-        return VariableAssignment(variable=variable.id, expression=value, type=type_)
+        return VariableAssignment(
+            variable=Identifier(variable.id), expression=value, type=type_
+        )
     elif isinstance(variable, ast.Attribute):
+        # noinspection PyTypeChecker
         property_access = _convert_expression(variable, False)
         assert isinstance(property_access, FunctionCall)
         return PropertyAssignment(property=property_access, expression=value)
@@ -194,7 +200,7 @@ def _convert_call(node: ast.Call) -> FunctionCall:
         raise InvalidDslError("Passing arguments to property calls is not supported.")
 
     return FunctionCall(
-        type=our_type, name=name, namespace=namespace, arguments=arguments
+        type=our_type, name=Identifier(name), namespace=namespace, arguments=arguments
     )
 
 
@@ -256,7 +262,7 @@ def _convert_expression(node: ast.expr, is_return: bool) -> Expression:
             )
         return FunctionCall(
             type=FunctionType.PROPERTY,
-            name=node.attr,
+            name=Identifier(node.attr),
             namespace=_convert_expression(node.value, is_return),
             arguments=[],
         )
@@ -267,7 +273,9 @@ def _convert_expression(node: ast.expr, is_return: bool) -> Expression:
         elif node.id == "Undefined":
             return serialize_from_python(None, AdvancedNothingTypes.UNDEFINED)
         elif node.id.isupper():
-            return FunctionCall(type=FunctionType.PROPERTY, name=node.id.lower())
+            return FunctionCall(
+                type=FunctionType.PROPERTY, name=Identifier(node.id.lower())
+            )
         else:
             return Identifier(node.id)
     elif isinstance(node, ast.List):

tested/dsl/translate_parser.py

@@ -33,6 +33,7 @@
 from tested.parsing import get_converter, suite_to_json
 from tested.serialisation import (
     BooleanType,
+    Identifier,
     NothingType,
     NumberType,
     ObjectKeyValuePair,
@@ -603,7 +604,7 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite:
 
     if namespace:
         assert isinstance(namespace, str)
-        return Suite(tabs=tabs, namespace=namespace)
+        return Suite(tabs=tabs, namespace=Identifier(namespace))
     else:
         return Suite(tabs=tabs)
 

tested/languages/bash/generators.py

@@ -249,7 +249,7 @@ def convert_encoder(values: list[Value]) -> str:
     for value in values:
         index_fun = unique_index_function()
         function_call = FunctionCall(
-            type=FunctionType.FUNCTION, name="send_value", arguments=[value]
+            type=FunctionType.FUNCTION, name=Identifier("send_value"), arguments=[value]
         )
         result += f"{convert_function_call(function_call, index_fun, [])}\n"
     return result

tested/languages/conventionalize.py

@@ -6,13 +6,15 @@
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Literal
 
+from tested.serialisation import Identifier
+
 if TYPE_CHECKING:
     from tested.languages import Language
 
 _logger = logging.getLogger(__name__)
 
 
-EXECUTION_PREFIX = "execution"
+EXECUTION_PREFIX = Identifier("execution")
 
 
 def camel_snake_case(what: str) -> str:
@@ -335,14 +337,17 @@ def upper_flat_case(what: str) -> str:
 
 
 def _conventionalize(
-    language: "Language", what: Conventionable, identifier: str
-) -> str:
+    language: "Language", what: Conventionable, identifier: Identifier
+) -> Identifier:
+    if identifier.is_raw:
+        return identifier
     conventions = language.naming_conventions()
     mapper = _case_mapping[conventions.get(what, "snake_case")]
-    return mapper(identifier)
+    result = mapper(identifier)
+    return Identifier(result)
 
 
-def conventionalize_class(language: "Language", class_name: str) -> str:
+def conventionalize_class(language: "Language", class_name: Identifier) -> Identifier:
     """
     Conventionalize a class name.
 
@@ -353,7 +358,9 @@ def conventionalize_class(language: "Language", class_name: str) -> str:
     return _conventionalize(language, "class", class_name)
 
 
-def conventionalize_function(language: "Language", function_name: str) -> str:
+def conventionalize_function(
+    language: "Language", function_name: Identifier
+) -> Identifier:
     """
     Conventionalize the name of a function.
 
@@ -364,7 +371,9 @@ def conventionalize_function(language: "Language", function_name: str) -> str:
     return _conventionalize(language, "function", function_name)
 
 
-def conventionalize_identifier(language: "Language", identifier: str) -> str:
+def conventionalize_identifier(
+    language: "Language", identifier: Identifier
+) -> Identifier:
     """
     Conventionalize the name of an identifier.
 
@@ -375,7 +384,9 @@ def conventionalize_identifier(language: "Language", identifier: str) -> str:
     return _conventionalize(language, "identifier", identifier)
 
 
-def conventionalize_global_identifier(language: "Language", identifier: str) -> str:
+def conventionalize_global_identifier(
+    language: "Language", identifier: Identifier
+) -> Identifier:
     """
     Conventionalize the name of a global identifier.
 
@@ -386,7 +397,9 @@ def conventionalize_global_identifier(language: "Language", identifier: str) ->
     return _conventionalize(language, "global_identifier", identifier)
 
 
-def conventionalize_namespace(language: "Language", namespace: str) -> str:
+def conventionalize_namespace(
+    language: "Language", namespace: Identifier
+) -> Identifier:
     """
     Conventionalize a namespace.
 
@@ -402,7 +415,9 @@ def conventionalize_namespace(language: "Language", namespace: str) -> str:
     return _conventionalize(language, "namespace", namespace)
 
 
-def conventionalize_property(language: "Language", property_name: str) -> str:
+def conventionalize_property(
+    language: "Language", property_name: Identifier
+) -> Identifier:
     """
     Conventionalize the name of a property.
 
@@ -414,7 +429,7 @@ def conventionalize_property(language: "Language", property_name: str) -> str:
     return _conventionalize(language, "property", property_name)
 
 
-def submission_name(language: "Language") -> str:
+def submission_name(language: "Language") -> Identifier:
     """
     :return: The name of a submission.
     """
@@ -429,11 +444,11 @@ def submission_file(language: "Language") -> str:
     return language.submission_file()
 
 
-def selector_name(language: "Language") -> str:
+def selector_name(language: "Language") -> Identifier:
     """
     :return: The name for the selector, conventionalized.
     """
-    return conventionalize_namespace(language, "selector")
+    return conventionalize_namespace(language, Identifier("selector"))
 
 
 def selector_file(language: "Language") -> str:
@@ -443,7 +458,7 @@ def selector_file(language: "Language") -> str:
     return language.with_extension(selector_name(language))
 
 
-def execution_name(language: "Language", execution_number: int) -> str:
+def execution_name(language: "Language", execution_number: int) -> Identifier:
     """
     Get the name of an execution. The name should be unique for the tab and
     execution number combination.
@@ -452,5 +467,5 @@ def execution_name(language: "Language", execution_number: int) -> str:
     :param execution_number: The number of the execution.
     :return: The name of the execution.
     """
-    name = f"{EXECUTION_PREFIX}_{execution_number}"
+    name = Identifier(f"{EXECUTION_PREFIX}_{execution_number}")
     return conventionalize_namespace(language, name)

tested/languages/preparation.py

@@ -204,7 +204,9 @@ def prepare_argument(
 def prepare_assignment(bundle: Bundle, assignment: Assignment) -> Assignment:
     if isinstance(assignment, VariableAssignment):
         if isinstance(assignment.type, VariableType):
-            class_type = conventionalize_class(bundle.language, assignment.type.data)
+            class_type = conventionalize_class(
+                bundle.language, Identifier(assignment.type.data)
+            )
             assignment = assignment.replace_type(VariableType(data=class_type))
 
         assignment = assignment.replace_variable(
@@ -309,7 +311,9 @@ def _create_handling_function(
         output.oracle, LanguageSpecificOracle
     ):
         evaluator = output.oracle.for_language(bundle.config.programming_language)
-        evaluator_name = conventionalize_namespace(lang_config, evaluator.file.stem)
+        evaluator_name = conventionalize_namespace(
+            lang_config, Identifier(evaluator.file.stem)
+        )
     else:
         evaluator_name = None
         evaluator = None
@@ -319,10 +323,13 @@ def generator(expression: Expression) -> Statement:
             output.oracle, LanguageSpecificOracle
         ):
             assert evaluator
+            assert evaluator_name
             arguments = [
                 PreparedFunctionCall(
                     type=FunctionType.FUNCTION,
-                    name=conventionalize_function(lang_config, evaluator.name),
+                    name=conventionalize_function(
+                        lang_config, Identifier(evaluator.name)
+                    ),
                     namespace=Identifier(evaluator_name),
                     arguments=[prepare_expression(bundle, expression)],
                     has_root_namespace=False,
@@ -335,7 +342,7 @@ def generator(expression: Expression) -> Statement:
 
         internal = PreparedFunctionCall(
             type=FunctionType.FUNCTION,
-            name=conventionalize_function(lang_config, function_name),
+            name=conventionalize_function(lang_config, Identifier(function_name)),
             arguments=[prepare_argument(bundle, arg) for arg in arguments],
             has_root_namespace=False,
         )

tested/oracles/programmed.py

@@ -98,18 +98,22 @@ def _evaluate_programmed(
 
     # The context in which to execute.
     global_env = {
-        "__tested_test__": utils,
-        "__tested_context__": ConvertedOracleContext.from_context(eval_bundle, context),
+        "t__tested_test__": utils,
+        "t__tested_context__": ConvertedOracleContext.from_context(
+            eval_bundle, context
+        ),
     }
-    exec("import sys\n" "sys.modules['evaluation_utils'] = __tested_test__", global_env)
+    exec(
+        "import sys\n" "sys.modules['evaluation_utils'] = t__tested_test__", global_env
+    )
     # Make the oracle available.
     exec(evaluator_code, global_env)
 
     # Since we pass a class value, we don't want to
     check_function_call = FunctionCall(
         type=FunctionType.FUNCTION,
-        name=oracle.function.name,
-        arguments=[Identifier("__tested_context__"), *oracle.arguments],
+        name=Identifier(oracle.function.name),
+        arguments=[Identifier("t__tested_context__"), *oracle.arguments],
     )
     literal_function_call = generate_statement(eval_bundle, check_function_call)