feat(mmap): implement reading and writing of mmap files

go.mod

@@ -4,12 +4,19 @@ go 1.22.1
 
 require (
 	github.com/consensys/gnark-crypto v0.12.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.8.2
+	golang.org/x/sys v0.9.0
 )
 
 require (
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -3,23 +3,41 @@ github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
 github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/air/eval.go

@@ -5,7 +5,7 @@ import (
 	"github.com/consensys/go-corset/pkg/table"
 )
 
-// EvalAt evaluates a column access at a given row in a trace, which returns the
+// EvalAt evaluates a column access at a given row in a mmap, which returns the
 // value at that row of the column in question or nil is that row is
 // out-of-bounds.
 func (e *ColumnAccess) EvalAt(k int, tbl table.Trace) *fr.Element {
@@ -24,29 +24,29 @@ func (e *ColumnAccess) EvalAt(k int, tbl table.Trace) *fr.Element {
 	return clone.Set(val)
 }
 
-// EvalAt evaluates a constant at a given row in a trace, which simply returns
+// EvalAt evaluates a constant at a given row in a mmap, which simply returns
 // that constant.
 func (e *Constant) EvalAt(k int, tbl table.Trace) *fr.Element {
 	var clone fr.Element
 	// Clone original value
 	return clone.Set(e.Value)
 }
 
-// EvalAt evaluates a sum at a given row in a trace by first evaluating all of
+// EvalAt evaluates a sum at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Add) EvalAt(k int, tbl table.Trace) *fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Add(l, r) }
 	return evalExprsAt(k, tbl, e.Args, fn)
 }
 
-// EvalAt evaluates a product at a given row in a trace by first evaluating all of
+// EvalAt evaluates a product at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Mul) EvalAt(k int, tbl table.Trace) *fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Mul(l, r) }
 	return evalExprsAt(k, tbl, e.Args, fn)
 }
 
-// EvalAt evaluates a subtraction at a given row in a trace by first evaluating all of
+// EvalAt evaluates a subtraction at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Sub) EvalAt(k int, tbl table.Trace) *fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Sub(l, r) }

pkg/air/expr.go

@@ -10,7 +10,7 @@ import (
 // Expressions at this level are split into those which can be arithmetised and
 // those which cannot.  The latter represent expressions which cannot be
 // expressed within a polynomial but can be computed externally (e.g. during
-// trace expansion).
+// mmap expansion).
 type Expr interface {
 	// EvalAt evaluates this expression in a given tabular context. Observe that
 	// if this expression is *undefined* within this context then it returns

pkg/binfile/constraint_set.go

@@ -21,7 +21,7 @@ type column struct {
 	// can be assigned to the same "register".
 	Register int
 	// Indicates the padding value (if given) to use when padding
-	// out a trace for this column.
+	// out a mmap for this column.
 	PaddingValue any `json:"padding_value"`
 	// Determines whether the type was marked with "@prove" or
 	// not.  Such types must be established by corset in some way
@@ -42,7 +42,7 @@ type column struct {
 	Computed bool
 	// Provides additional information about whether this column
 	// is computed or not.  A "Commitment" kind indicates a
-	// user-defined columns (i.e is directly filled from trace
+	// user-defined columns (i.e is directly filled from mmap
 	// files); a "Computed" column is filled by a given function;
 	// an "Expression" kind indicates a column whose values are
 	// computed from an expresion known at compile time.  As for

pkg/cmd/compute.go

@@ -12,8 +12,8 @@ import (
 // computeCmd represents the compute command
 var computeCmd = &cobra.Command{
 	Use:   "compute",
-	Short: "Given a set of constraints and a trace file, fill the computed columns.",
-	Long:  `Given a set of constraints and a trace file, fill the computed columns.`,
+	Short: "Given a set of constraints and a mmap file, fill the computed columns.",
+	Long:  `Given a set of constraints and a mmap file, fill the computed columns.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		file := args[0]
 		fmt.Printf("Reading JSON bin file: %s\n", file)

pkg/hir/eval.go

@@ -5,7 +5,7 @@ import (
 	"github.com/consensys/go-corset/pkg/table"
 )
 
-// EvalAllAt evaluates a column access at a given row in a trace, which returns the
+// EvalAllAt evaluates a column access at a given row in a mmap, which returns the
 // value at that row of the column in question or nil is that row is
 // out-of-bounds.
 func (e *ColumnAccess) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
@@ -23,29 +23,29 @@ func (e *ColumnAccess) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	return []*fr.Element{clone.Set(val)}
 }
 
-// EvalAllAt evaluates a constant at a given row in a trace, which simply returns
+// EvalAllAt evaluates a constant at a given row in a mmap, which simply returns
 // that constant.
 func (e *Constant) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	var clone fr.Element
 	// Clone original value
 	return []*fr.Element{clone.Set(e.Val)}
 }
 
-// EvalAllAt evaluates a sum at a given row in a trace by first evaluating all of
+// EvalAllAt evaluates a sum at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Add) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Add(l, r) }
 	return evalExprsAt(k, tbl, e.Args, fn)
 }
 
-// EvalAllAt evaluates a product at a given row in a trace by first evaluating all of
+// EvalAllAt evaluates a product at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Mul) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Mul(l, r) }
 	return evalExprsAt(k, tbl, e.Args, fn)
 }
 
-// EvalAllAt evaluates a conditional at a given row in a trace by first evaluating
+// EvalAllAt evaluates a conditional at a given row in a mmap by first evaluating
 // its condition at that row.  If that condition is zero then the true branch
 // (if applicable) is evaluated; otherwise if the condition is non-zero then
 // false branch (if applicable) is evaluated).  If the branch to be evaluated is
@@ -66,7 +66,7 @@ func (e *IfZero) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	return vals
 }
 
-// EvalAllAt evaluates a list at a given row in a trace by evaluating each of its
+// EvalAllAt evaluates a list at a given row in a mmap by evaluating each of its
 // arguments at that row.
 func (e *List) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	vals := make([]*fr.Element, 0)
@@ -95,7 +95,7 @@ func (e *Normalise) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	return vals
 }
 
-// EvalAllAt evaluates a subtraction at a given row in a trace by first evaluating all of
+// EvalAllAt evaluates a subtraction at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Sub) EvalAllAt(k int, tbl table.Trace) []*fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Sub(l, r) }

pkg/hir/schema.go

@@ -132,8 +132,8 @@ func (p *Schema) AddPropertyAssertion(handle string, expr mir.Expr) {
 	p.assertions = append(p.assertions, table.NewPropertyAssertion[mir.Expr](handle, expr))
 }
 
-// Accepts determines whether this schema will accept a given trace.  That
-// is, whether or not the given trace adheres to the schema.  A trace can fail
+// Accepts determines whether this schema will accept a given mmap.  That
+// is, whether or not the given mmap adheres to the schema.  A mmap can fail
 // to adhere to the schema for a variety of reasons, such as having a constraint
 // which does not hold.
 func (p *Schema) Accepts(trace table.Trace) error {
@@ -157,7 +157,7 @@ func (p *Schema) Accepts(trace table.Trace) error {
 	return nil
 }
 
-// ExpandTrace expands a given trace according to this schema.
+// ExpandTrace expands a given mmap according to this schema.
 func (p *Schema) ExpandTrace(tr table.Trace) error {
 	// Insert initial padding row
 	table.PadTrace(1, tr)

pkg/mir/eval.go

@@ -5,7 +5,7 @@ import (
 	"github.com/consensys/go-corset/pkg/table"
 )
 
-// EvalAt evaluates a column access at a given row in a trace, which returns the
+// EvalAt evaluates a column access at a given row in a mmap, which returns the
 // value at that row of the column in question or nil is that row is
 // out-of-bounds.
 func (e *ColumnAccess) EvalAt(k int, tbl table.Trace) *fr.Element {
@@ -23,15 +23,15 @@ func (e *ColumnAccess) EvalAt(k int, tbl table.Trace) *fr.Element {
 	return clone.Set(val)
 }
 
-// EvalAt evaluates a constant at a given row in a trace, which simply returns
+// EvalAt evaluates a constant at a given row in a mmap, which simply returns
 // that constant.
 func (e *Constant) EvalAt(k int, tbl table.Trace) *fr.Element {
 	var clone fr.Element
 	// Clone original value
 	return clone.Set(e.Value)
 }
 
-// EvalAt evaluates a sum at a given row in a trace by first evaluating all of
+// EvalAt evaluates a sum at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Add) EvalAt(k int, tbl table.Trace) *fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Add(l, r) }
@@ -59,7 +59,7 @@ func (e *Normalise) EvalAt(k int, tbl table.Trace) *fr.Element {
 	return val
 }
 
-// EvalAt evaluates a subtraction at a given row in a trace by first evaluating all of
+// EvalAt evaluates a subtraction at a given row in a mmap by first evaluating all of
 // its arguments at that row.
 func (e *Sub) EvalAt(k int, tbl table.Trace) *fr.Element {
 	fn := func(l *fr.Element, r *fr.Element) { l.Sub(l, r) }

pkg/mir/schema.go

@@ -98,8 +98,8 @@ func (p *Schema) AddPropertyAssertion(handle string, expr Expr) {
 	p.assertions = append(p.assertions, table.NewPropertyAssertion(handle, expr))
 }
 
-// Accepts determines whether this schema will accept a given trace.  That
-// is, whether or not the given trace adheres to the schema.  A trace can fail
+// Accepts determines whether this schema will accept a given mmap.  That
+// is, whether or not the given mmap adheres to the schema.  A mmap can fail
 // to adhere to the schema for a variety of reasons, such as having a constraint
 // which does not hold.
 func (p *Schema) Accepts(trace table.Trace) error {
@@ -176,7 +176,7 @@ func lowerColumnToAir(c *table.DataColumn[table.Type], schema *air.Schema) {
 // Lower a permutation to the AIR level.  This has quite a few
 // effects.  Firstly, permutation constraints are added for all of the
 // new columns.  Secondly, sorting constraints (and their associated
-// synthetic columns) must also be added.  Finally, a trace
+// synthetic columns) must also be added.  Finally, a mmap
 // computation is required to ensure traces are correctly expanded to
 // meet the requirements of a sorted permutation.
 func lowerPermutationToAir(c Permutation, mirSchema *Schema, airSchema *air.Schema) {
@@ -215,7 +215,7 @@ func lowerPermutationToAir(c Permutation, mirSchema *Schema, airSchema *air.Sche
 	}
 }
 
-// ExpandTrace expands a given trace according to this schema.
+// ExpandTrace expands a given mmap according to this schema.
 func (p *Schema) ExpandTrace(tr table.Trace) error {
 	// Insert initial padding row
 	table.PadTrace(1, tr)

pkg/mmap/block_device.go

@@ -0,0 +1,99 @@
+package mmap
+
+import (
+	"errors"
+	"io"
+	"runtime/debug"
+	"syscall"
+
+	pkgErrors "github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+// BlockDevice represents a mmap block device holding a reference to a file descriptor.
+type BlockDevice struct {
+	FileDescriptor int
+	Data           []byte
+}
+
+// NewBlockDevice creates a BlockDevice from a file
+// descriptor referring either to a regular file or UNIX device node. To
+// speed up reads, a memory map is used.
+func NewBlockDevice(fileDescriptor, sizeBytes int) (*BlockDevice, error) {
+	data, err := unix.Mmap(fileDescriptor, 0, sizeBytes, syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, pkgErrors.Wrap(err, "failed to memory map block device")
+	}
+
+	return &BlockDevice{
+		FileDescriptor: fileDescriptor,
+		Data:           data,
+	}, nil
+}
+
+// ReadAt reads through the memory map at a given offset.
+func (bd *BlockDevice) ReadAt(p []byte, off int64) (n int, err error) {
+	// Let read actions go through the memory map to prevent system
+	// call overhead for commonly requested objects.
+	if off < 0 {
+		return 0, syscall.EINVAL
+	}
+
+	if off > int64(len(bd.Data)) {
+		return 0, io.EOF
+	}
+	// Install a page fault handler, so that I/O errors against the
+	// memory map (e.g., due to disk failure) don't cause us to
+	// crash.
+	old := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(old)
+
+		if recover() != nil {
+			err = errors.New("page fault occurred while reading from memory map")
+		}
+	}()
+
+	n = copy(p, bd.Data[off:])
+	if n < len(p) {
+		err = io.EOF
+	}
+
+	return
+}
+
+// WriteAt writes at a given offset.
+func (bd *BlockDevice) WriteAt(p []byte, off int64) (int, error) {
+	// Let write actions go through the file descriptor. Doing so
+	// yields better performance, as writes through a memory map
+	// would trigger a page fault that causes data to be read.
+	//
+	// The pwrite() system call cannot return a size and error at
+	// the same time. If an error occurs after one or more bytes are
+	// written, it returns the size without an error (a "short
+	// write"). As WriteAt() must return an error in those cases, we
+	// must invoke pwrite() repeatedly.
+	//
+	// TODO: Maybe it makes sense to let unaligned writes that would
+	// trigger reads anyway to go through the memory map?
+	nTotal := 0
+
+	for len(p) > 0 {
+		n, err := unix.Pwrite(bd.FileDescriptor, p, off)
+		nTotal += n
+
+		if err != nil {
+			return nTotal, err
+		}
+
+		p = p[n:]
+		off += int64(n)
+	}
+
+	return nTotal, nil
+}
+
+// Sync synchronizes a file's in-core state with storage device.
+func (bd *BlockDevice) Sync() error {
+	return unix.Fsync(bd.FileDescriptor)
+}