ads dynamic length slices

ssz/marshal.go

@@ -191,11 +191,28 @@ func (d *DynSsz) marshalSlice(sourceType reflect.Type, sourceValue reflect.Value
 		fieldType = fieldType.Elem()
 	}
 
-	sliceLen := sourceValue.Len()
+	isDynSlice := false
+	if len(sizeHints) > 1 && sizeHints[1].dynamic {
+		isDynSlice = true
+	} else {
+		size, _, err := d.getSszSize(fieldType, childSizeHints)
+		if err != nil {
+			return nil, err
+		}
+		if size < 0 {
+			isDynSlice = true
+		}
+	}
+
+	if isDynSlice {
+		return d.marshalDynamicSlice(sourceType, sourceValue, buf, childSizeHints, idt)
+	}
+
 	if fieldType == byteType {
 		// shortcut for performance: use append on []byte arrays
 		buf = append(buf, sourceValue.Bytes()...)
 	} else {
+		sliceLen := sourceValue.Len()
 		for i := 0; i < sliceLen; i++ {
 			itemVal := sourceValue.Index(i)
 			if fieldIsPtr {
@@ -212,3 +229,42 @@ func (d *DynSsz) marshalSlice(sourceType reflect.Type, sourceValue reflect.Value
 
 	return buf, nil
 }
+
+func (d *DynSsz) marshalDynamicSlice(sourceType reflect.Type, sourceValue reflect.Value, buf []byte, sizeHints []sszSizeHint, idt int) ([]byte, error) {
+	sliceLen := sourceValue.Len()
+	startOffset := len(buf)
+	offsetBuf := make([]byte, 4*sliceLen)
+	buf = append(buf, offsetBuf...)
+
+	fieldType := sourceType.Elem()
+	fieldIsPtr := fieldType.Kind() == reflect.Ptr
+	if fieldIsPtr {
+		fieldType = fieldType.Elem()
+	}
+
+	offset := 4 * sliceLen
+	bufLen := len(buf)
+
+	for i := 0; i < sliceLen; i++ {
+		itemVal := sourceValue.Index(i)
+		if fieldIsPtr {
+			itemVal = itemVal.Elem()
+		}
+
+		newBuf, err := d.marshalType(fieldType, itemVal, buf, sizeHints, idt+2)
+		if err != nil {
+			return nil, err
+		}
+		newBufLen := len(newBuf)
+		buf = newBuf
+
+		offsetBuf := make([]byte, 4)
+		binary.LittleEndian.PutUint32(offsetBuf, uint32(offset))
+		copy(buf[startOffset+(i*4):startOffset+((i+1)*4)], offsetBuf)
+
+		offset += newBufLen - bufLen
+		bufLen = newBufLen
+	}
+
+	return buf, nil
+}

ssz/sszsize.go

@@ -206,11 +206,27 @@ func (d *DynSsz) getSszValueSize(targetType reflect.Type, targetValue reflect.Va
 		sliceLen := targetValue.Len()
 
 		if sliceLen > 0 {
-			size, err := d.getSszValueSize(fieldType, targetValue.Index(0))
+			fieldTypeSize, _, err := d.getSszSize(fieldType, nil)
 			if err != nil {
 				return 0, err
 			}
-			staticSize = size * sliceLen
+
+			if fieldTypeSize < 0 {
+				// dyn size slice
+				for i := 0; i < sliceLen; i++ {
+					size, err := d.getSszValueSize(fieldType, targetValue.Index(i))
+					if err != nil {
+						return 0, err
+					}
+					staticSize += size + 4
+				}
+			} else {
+				size, err := d.getSszValueSize(fieldType, targetValue.Index(0))
+				if err != nil {
+					return 0, err
+				}
+				staticSize = size * sliceLen
+			}
 		}
 	case reflect.Bool:
 		staticSize = 1

ssz/test/main.go

@@ -15,10 +15,18 @@ import (
 )
 
 func main() {
-	body, _ := ioutil.ReadFile("state2.ssz")
+	body, _ := ioutil.ReadFile("block-min.ssz")
+
+	d := ssz.NewDynSsz(map[string]any{
+		"SYNC_COMMITTEE_SIZE":          uint64(32),
+		"SYNC_COMMITTEE_SUBNET_COUNT":  uint64(4),
+		"EPOCHS_PER_HISTORICAL_VECTOR": uint64(64),
+		"SLOTS_PER_HISTORICAL_ROOT":    uint64(64),
+	})
+	d.NoFastSsz = true
 
 	test1(body)
-	test2(body)
+	test2(d, body)
 
 	f, _ := os.Create("mem.pprof")
 	pprof.WriteHeapProfile(f)
@@ -48,7 +56,7 @@ func test1(body []byte) {
 
 	fmt.Printf("\nfastssz / go-eth2-client\n")
 
-	t := new(deneb.BeaconState)
+	t := new(deneb.SignedBeaconBlock)
 	err := t.UnmarshalSSZ(body)
 	if err != nil {
 		fmt.Printf("error: %v\n", err)
@@ -60,7 +68,7 @@ func test1(body []byte) {
 
 	//root, _ := t.HashTreeRoot()
 	//fmt.Printf("state root: 0x%x\n", root)
-	fmt.Printf("gvr: 0x%x\n", t.GenesisValidatorsRoot)
+	//fmt.Printf("gvr: 0x%x\n", t.GenesisValidatorsRoot)
 
 	_, err = t.MarshalSSZ()
 	if err != nil {
@@ -69,7 +77,7 @@ func test1(body []byte) {
 	}
 }
 
-func test2(body []byte) {
+func test2(d *ssz.DynSsz, body []byte) {
 	start := time.Now()
 	defer func() {
 		elapsed := time.Since(start)
@@ -78,9 +86,7 @@ func test2(body []byte) {
 
 	fmt.Printf("\npk's dynamic ssz\n")
 
-	d := ssz.NewDynSsz(map[string]any{})
-
-	t := new(deneb.BeaconState)
+	t := new(deneb.SignedBeaconBlock)
 	err := d.UnmarshalSSZ(t, body)
 	if err != nil {
 		fmt.Printf("error: %v\n", err)
@@ -92,23 +98,22 @@ func test2(body []byte) {
 
 	//root, _ := t.HashTreeRoot()
 	//fmt.Printf("state root: 0x%x\n", root)
-	fmt.Printf("gvr: 0x%x\n", t.GenesisValidatorsRoot)
+	//fmt.Printf("gvr: 0x%x\n", t.GenesisValidatorsRoot)
 
-	_, err = d.MarshalSSZ(t)
+	buf, err := d.MarshalSSZ(t)
 	if err != nil {
 		fmt.Printf("error: %v\n", err)
 		return
 	}
 
-	/*
-		if len(buf) != len(body) {
-			fmt.Printf("size mismatch: %v / %v\n", len(buf), len(body))
-		}
-		for i := 0; i < len(body); i++ {
-			if body[i] != buf[i] {
-				fmt.Printf("ssz mismatch: %v : %v / %v\n", i, buf[i], body[i])
-				break
-			}
+	if len(buf) != len(body) {
+		fmt.Printf("size mismatch: %v / %v\n", len(buf), len(body))
+	}
+	for i := 0; i < len(body); i++ {
+		if body[i] != buf[i] {
+			fmt.Printf("ssz mismatch: %v : %v / %v\n", i, buf[i], body[i])
+			break
 		}
-	*/
+	}
+
 }

ssz/unmarshal.go

@@ -260,7 +260,8 @@ func (d *DynSsz) unmarshalSlice(targetType reflect.Type, targetValue reflect.Val
 	}
 
 	if sliceLen == 0 && len(ssz) > 0 {
-		return 0, fmt.Errorf("cannot deteriminate length of dynamic slice")
+		// dynamic size slice
+		return d.unmarshalDynamicSlice(targetType, targetValue, ssz, childSizeHints, idt)
 	}
 
 	//fmt.Printf("new slice %v  %v\n", fieldType.Name(), sliceLen)
@@ -305,3 +306,66 @@ func (d *DynSsz) unmarshalSlice(targetType reflect.Type, targetValue reflect.Val
 
 	return consumedBytes, nil
 }
+
+func (d *DynSsz) unmarshalDynamicSlice(targetType reflect.Type, targetValue reflect.Value, ssz []byte, sizeHints []sszSizeHint, idt int) (int, error) {
+	firstOffset := fastssz.ReadOffset(ssz[0:4])
+	sliceLen := int(firstOffset / 4)
+
+	sliceOffsets := make([]int, sliceLen)
+	sliceOffsets[0] = int(firstOffset)
+	for i := 1; i < sliceLen; i++ {
+		sliceOffsets[i] = int(fastssz.ReadOffset(ssz[i*4 : (i+1)*4]))
+	}
+
+	fieldType := targetType.Elem()
+	fieldIsPtr := fieldType.Kind() == reflect.Ptr
+	if fieldIsPtr {
+		fieldType = fieldType.Elem()
+	}
+
+	for i := 0; i < sliceLen; i++ {
+
+	}
+
+	//fmt.Printf("new dynamic slice %v  %v\n", fieldType.Name(), sliceLen)
+	newValue := reflect.MakeSlice(targetType, sliceLen, sliceLen)
+	targetValue.Set(newValue)
+
+	offset := int(firstOffset)
+	if sliceLen > 0 {
+		for i := 0; i < sliceLen; i++ {
+			var itemVal reflect.Value
+			if fieldIsPtr {
+				//fmt.Printf("new slice item %v\n", fieldType.Name())
+				itemVal = reflect.New(fieldType).Elem()
+				newValue.Index(i).Set(itemVal.Addr())
+			} else {
+				itemVal = newValue.Index(i)
+			}
+
+			startOffset := sliceOffsets[i]
+			endOffset := 0
+			if i == sliceLen-1 {
+				endOffset = len(ssz)
+			} else {
+				endOffset = sliceOffsets[i+1]
+			}
+			itemSize := endOffset - startOffset
+
+			itemSsz := ssz[startOffset:endOffset]
+
+			consumed, err := d.unmarshalType(fieldType, itemVal, itemSsz, sizeHints, idt+2)
+			if err != nil {
+				return 0, err
+			}
+			if consumed != itemSize {
+				return 0, fmt.Errorf("dynamic slice item did not consume expected ssz range (consumed: %v, expected: %v)", consumed, itemSize)
+			}
+
+			offset += itemSize
+		}
+	}
+
+	return offset, nil
+
+}