Progress Towards First-Class Module System

Now, when using the claro_library Bazel macro (which needs to be upgraded to a rule) as following to request compilation of a Claro Module from a .claro_module_api file and some .claro source files:

```
claro_library(
  name = "foo_module",
  module_api_file = "FooModule.claro_module_api",
  src = ["FooImpl.claro", ...]
)
```

Claro will compile the given files into a .claro_module which (in the future) will be able to be depended upon by other `claro_library(...)` targets or more importantly by a `claro_binary(...)` target in order to compose a complete executable program from a DAG of modules.

For a bit of added detail, this .claro_module file actually just contains a serialized proto message of SerializedClaroModule.proto for convenient serialization/deserialization. This choice to use proto for this purpose will, I hope, turn out to be a good choice in the long run as I can eventually (once backwards compatibility concerns apply) actually utilize Proto's powerful backwards compatibility features.

BUILD

@@ -4,6 +4,10 @@ alias(
  name = "guava",
  actual = "@maven//:com_google_guava_guava",
 )
+alias(
+ name = "protobuf",
+ actual = "@com_google_protobuf_protobuf_java//:com_google_protobuf_protobuf_java",
+)
 alias(
  name = "apache_commons_text",
  actual = "@maven//:org_apache_commons_commons_text",

WORKSPACE

@@ -28,6 +28,18 @@ http_archive(
 load("@jflex_rules//jflex:deps.bzl", "JFLEX_ARTIFACTS")
 load("@jflex_rules//third_party:third_party_deps.bzl", "THIRD_PARTY_ARTIFACTS")
 
+http_archive(
+ name = "rules_proto",
+ sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
+ strip_prefix = "rules_proto-5.3.0-21.7",
+ urls = [
+ "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz",
+ ],
+)
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+rules_proto_dependencies()
+rules_proto_toolchains()
+
 # See this documentation to understand how fetching Maven deps works in Bazel:
 # https://github.com/bazelbuild/rules_jvm_external
 # When you add a new maven dep run the following command to update new deps:

src/java/com/claro/claro_build_rules.bzl

@@ -73,7 +73,12 @@ def gen_claro_builtin_java_deps_jar():
 def claro_library(name, src, module_api_file = None, java_name = None, claro_compiler_name = DEFAULT_CLARO_NAME, debug = False):
  if module_api_file and java_name:
  fail("claro_library: java_name must *not* be set when compiling a Module as signalled by providing a module_api_file.")
- isModule = module_api_file != None
+ if module_api_file:
+ if not module_api_file.endswith(".claro_module_api"):
+ fail("claro_library: Provided module_api_file must use .claro_module_api extension.")
+ isModule = True
+ else:
+ isModule = False
  if isModule:
  java_name = ""
  hasMultipleSrcs = str(type(src)) == "list"
@@ -103,6 +108,13 @@ def claro_library(name, src, module_api_file = None, java_name = None, claro_com
  claro_compiler_name,
  "false" if debug else "true", # --silent
  java_name, # --classname
+ # TODO(steving) Once this macro is migrated to being a full on Bazel "rule", swap this DEFAULT_PACKAGE w/
+ # TODO(steving) ctx.workspace_name instead.
+ # TODO(steving) All Bazel workspace names must be formatted as "Java-package-style" strings. This is a super convenient
+ # TODO(steving) way for this macro to automatically determine a "project_package" that should already be intended to be
+ # TODO(steving) globally (internet-wide) unique.
+ # TODO(steving) See: https://bazel.build/rules/lib/globals/workspace#parameters_3
+ # ctx.workspace_name.replace('_', '.').replace('-', '.'), # --package
  DEFAULT_PACKAGE_PREFIX, # --package
  # Here, construct a totally unique name for this particular module. Since we're using Bazel, I have the
  # guarantee that this RULEDIR+target name is globally unique across the entire project.
@@ -112,7 +124,7 @@ def claro_library(name, src, module_api_file = None, java_name = None, claro_com
  tools = [
  "//src/java/com/claro:claro_compiler_binary_deploy.jar",
  ],
- outs = [(java_name if java_name else module_api_file[:-len(".claro_module_api")]) + ".java"]
+ outs = [java_name + ".java" if java_name else module_api_file[:-len("_api")]]
  )
 
 def gen_claro_compiler(name = DEFAULT_CLARO_NAME):

src/java/com/claro/compiler_backends/java_source/BUILD

@@ -8,6 +8,7 @@ java_library(
  deps = [
  "//:autovalue",
  "//:guava",
+ "//:protobuf",
  "//src/java/com/claro:" + DEFAULT_CLARO_NAME + "_java_parser",
  "//src/java/com/claro:claro_parser_exception",
  "//src/java/com/claro/compiler_backends:compiler_backend",
@@ -18,6 +19,7 @@ java_library(
  "//src/java/com/claro/intermediate_representation:program_node",
  "//src/java/com/claro/intermediate_representation/expressions:expr",
  "//src/java/com/claro/module_system:module_api_parser_util",
+ "//src/java/com/claro/module_system/module_serialization/proto:serialized_claro_module_java_proto",
  "//src/java/com/claro/stdlib",
  ],
 )

src/java/com/claro/compiler_backends/java_source/JavaSourceCompilerBackend.java

@@ -12,10 +12,13 @@
 import com.claro.intermediate_representation.Target;
 import com.claro.intermediate_representation.expressions.Expr;
 import com.claro.module_system.ModuleApiParserUtil;
+import com.claro.module_system.module_serialization.proto.SerializedClaroModule;
 import com.claro.stdlib.StdLibUtil;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
 import com.google.common.io.CharSource;
+import com.google.protobuf.ByteString;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -41,20 +44,20 @@ public JavaSourceCompilerBackend(String... args) {
  if (args.length >= 2) {
  // args[1] holds the generated classname.
  this.GENERATED_CLASSNAME =
- Optional.of(args[1].substring("--classname=" .length())).map(n -> n.equals("") ? null : n);
+ Optional.of(args[1].substring("--classname=".length())).map(n -> n.equals("") ? null : n);
  // args[2] holds the flag for package...
- String packageArg = args[2].substring("--package=" .length());
- this.PACKAGE_STRING = Optional.of(packageArg.equals("") ? "" : "package " + packageArg + ";\n\n");
+ String packageArg = args[2].substring("--package=".length());
+ this.PACKAGE_STRING = Optional.of(packageArg);
  // args[3] is an *OPTIONAL* flag holding the list of files in this Claro module that should be read in instead of
  // reading a single Claro file's contents from stdin.
  if (args.length >= 4) {
  this.SRCS =
- ImmutableList.copyOf(args[3].substring("--srcs=" .length()).split(","))
+ ImmutableList.copyOf(args[3].substring("--srcs=".length()).split(","))
  .stream()
  .map(f -> SrcFile.forFilenameAndPath(f.substring(f.lastIndexOf('/') + 1, f.lastIndexOf('.')), f))
  .collect(ImmutableList.toImmutableList());
  if (args.length >= 5) {
- this.OPTIONAL_UNIQUE_MODULE_NAME = Optional.of(args[4].substring("--module_unique_prefix=" .length()));
+ this.OPTIONAL_UNIQUE_MODULE_NAME = Optional.of(args[4].substring("--module_unique_prefix=".length()));
  } else {
  this.OPTIONAL_UNIQUE_MODULE_NAME = Optional.empty();
  }
@@ -150,14 +153,15 @@ private Node.GeneratedJavaSource checkTypesAndGenJavaSourceForSrcFiles(
  nonMainSrcFiles.stream()
  .map(f -> {
  ClaroParser currNonMainSrcFileParser = getParserForSrcFile(f);
- this.PACKAGE_STRING.ifPresent(s -> currNonMainSrcFileParser.package_string = s);
+ this.PACKAGE_STRING.ifPresent(
+ s -> currNonMainSrcFileParser.package_string = s.equals("") ? "" : "package " + s + ";\n\n");
  return currNonMainSrcFileParser;
  })
  .collect(ImmutableList.toImmutableList());
  Optional<ModuleApiParser> optionalModuleApiParser =
  optionalModuleApiSrcFile.map(this::getModuleApiParserForSrcFile);
  ClaroParser mainSrcFileParser = getParserForSrcFile(mainSrcFile);
- this.PACKAGE_STRING.ifPresent(s -> mainSrcFileParser.package_string = s);
+ this.PACKAGE_STRING.ifPresent(s -> mainSrcFileParser.package_string = s.equals("") ? "" : "package " + s + ";\n\n");
 
  try {
  // Parse the non-main src files first.
@@ -181,7 +185,22 @@ private Node.GeneratedJavaSource checkTypesAndGenJavaSourceForSrcFiles(
  mainSrcFileParser.errorsFound +
  nonMainSrcFileParsers.stream().map(p -> p.errorsFound).reduce(Integer::sum).orElse(0);
  if (totalParserErrorsFound == 0 && Expr.typeErrorsFound.isEmpty() && ProgramNode.miscErrorsFound.isEmpty()) {
- System.out.println(generateTargetOutputRes);
+ if (optionalModuleApiParser.isPresent()) {
+ // Here, we were asked to compile a non-executable Claro Module, rather than an executable Claro program. So,
+ // we need to populate and emit a SerializedClaroModule proto that can be used as a dep for other Claro
+ // Modules/programs.
+ serializeClaroModule(
+ this.PACKAGE_STRING.get(),
+ this.OPTIONAL_UNIQUE_MODULE_NAME.get(),
+ optionalModuleApiSrcFile.get(),
+ generateTargetOutputRes,
+ nonMainSrcFiles
+ );
+ } else {
+ // Here, we were simply asked to codegen an executable Claro program. Output the codegen'd Java source to
+ // stdout directly where it will be piped by Claro's Bazel rules into the appropriate .java file.
+ System.out.println(generateTargetOutputRes);
+ }
  System.exit(0);
  } else {
  ClaroParser.errorMessages.forEach(Runnable::run);
@@ -222,6 +241,40 @@ private Node.GeneratedJavaSource checkTypesAndGenJavaSourceForSrcFiles(
  "Internal Compiler Error! Should be unreachable. JavaSourceCompilerBackend failed to exit with explicit error code.");
  }
 
+ private static void serializeClaroModule(
+ String projectPackage,
+ String uniqueModuleName,
+ SrcFile moduleApiSrcFile,
+ StringBuilder moduleCodegen,
+ ImmutableList<SrcFile> moduleImplFiles) throws IOException {
+ SerializedClaroModule.Builder serializedClaroModuleBuilder =
+ SerializedClaroModule.newBuilder()
+ .setModuleDescriptor(
+ SerializedClaroModule.UniqueModuleDescriptor.newBuilder()
+ .setProjectPackage(projectPackage)
+ .setUniqueModuleName(uniqueModuleName))
+ .setModuleApiFile(
+ SerializedClaroModule.ClaroSourceFile.newBuilder()
+ .setOriginalFilename(moduleApiSrcFile.getFilename())
+ .setOriginalFilenameBytes(
+ ByteString.copyFrom(ByteStreams.toByteArray(moduleApiSrcFile.getFileInputStream()))))
+ .setStaticJavaCodegen(
+ ByteString.copyFrom(moduleCodegen.toString().getBytes(StandardCharsets.UTF_8)));
+ for (SrcFile moduleImplFile : moduleImplFiles) {
+ serializedClaroModuleBuilder.addModuleImplFiles(
+ SerializedClaroModule.ClaroSourceFile.newBuilder()
+ .setOriginalFilename(moduleImplFile.getFilename())
+ .setOriginalFilenameBytes(
+ ByteString.copyFrom(ByteStreams.toByteArray(moduleImplFile.getFileInputStream()))));
+ }
+
+ // Finally write the proto message to stdout where Claro's Bazel rules will pipe this output into the appropriate
+ // .claro_module output file.
+ serializedClaroModuleBuilder
+ .build()
+ .writeDelimitedTo(System.out);
+ }
+
  private void warnNumErrorsFound(int totalParserErrorsFound) {
  int totalErrorsFound = totalParserErrorsFound + Expr.typeErrorsFound.size() + ProgramNode.miscErrorsFound.size();
  System.err.println(Math.max(totalErrorsFound, 1) + " Error" + (totalErrorsFound > 1 ? "s" : ""));

src/java/com/claro/module_system/module_serialization/proto/BUILD

@@ -0,0 +1,14 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+java_proto_library(
+ name = "serialized_claro_module_java_proto",
+ deps = [":serialized_claro_module_proto"],
+ visibility = [
+ "//src/java/com/claro/compiler_backends:__subpackages__",
+ ]
+)
+
+proto_library(
+ name = "serialized_claro_module_proto",
+ srcs = ["SerializedClaroModule.proto"],
+)

src/java/com/claro/module_system/module_serialization/proto/SerializedClaroModule.proto

@@ -0,0 +1,54 @@
+syntax = "proto3";
+
+package claro.module_system.module_serialization;
+
+option java_multiple_files = true;
+option java_package = "com.claro.module_system.module_serialization.proto";
+
+// Perhaps this may change going forward, however, the fact is that this proto is used as the intermediate format for
+// incremental compilation, so I'd much prefer to prioritize (de)serialization time to have a positive effect on compile
+// times. I don't think any of the choices made here so far lend themselves to much in the way of great disk utilization
+// so I won't pretend that's the case here just yet. Remember that this affects dev machines not deploy binary sizes.
+option optimize_for = SPEED;
+
+message SerializedClaroModule {
+ message ClaroSourceFile {
+ string original_filename = 1;
+ bytes source_utf8 = 2;
+ }
+
+ message UniqueModuleDescriptor {
+ // Claro doesn't utilize a "package visibility" system like Java does, instead it assumes that you'll control code
+ // visibility using Bazel's builtin target visibility controls. However, it does need some mechanism for ensuring
+ // that cross-project .claro_modules do not produce artifacts that collide in the same Java namespace. So, Claro
+ // will utilize a concept of "project package" as a global (internet-wide) disambiguator. When building using
+ // Claro's Bazel rules, this string will be automatically derived from the Bazel workspace's name. This is in an
+ // effort towards utilizing Bazel's native Bazelmod as Claro's "package manager".
+ string project_package = 1;
+ // When building using Claro's Bazel rules, this string will be automatically formatted to be something like
+ // `src$java$com$claro$claro_programs$module_test` for a `claro_library(name = "module_test", ...)` located at the
+ // bazel path //src/java/com/claro/claro_programs:module_test. This ensures that this Module name is unique across
+ // the entire Bazel project in which this module is being compiled.
+ string unique_module_name = 2;
+ }
+
+ UniqueModuleDescriptor module_descriptor = 1;
+
+ // This field contains the actual source of the given .claro_module_api file. This will be necessary for dependents to
+ // be able to determine the bindings exported for them to make use of.
+ ClaroSourceFile module_api_file = 2;
+
+ // This field contains the codegen pertaining to any static definitions that are guaranteed to be provided by this
+ // module. In less opaque terms, this contains codegen for any non-generic procedure definitions that are exported by
+ // this module. This *does not* include any codegen for generic procedure monomorphizations (except for those
+ // monomorphizations that are explicitly called from exported procedures *within* this module). Later on, when this
+ // module is depended on in a Claro Binary, the concrete monomorphizations needed by the overall will be codegen'd at
+ // the last minute.
+ bytes static_java_codegen = 3;
+
+ // As Claro will need to postpone codegen of generic procedure monomorphizations until after the entire program is
+ // assembled into a Claro Binary, this serialized module actually needs to maintain the original Claro source files
+ // that contain all exported procedure implementations. This way, once all concrete monomorphizations are known, it's
+ // possible to re-parse the source file to find the necessary generic procedure defs to codegen monomorphizations for.
+ repeated ClaroSourceFile module_impl_files = 4;
+}