add ability to work with encoded files

README.md

@@ -14,11 +14,15 @@ Simple tool to extract all Star Citizen keybinds to a CVS file
 Let's assume what the game data located at `C:\Program Files\Star Citizen\LIVE`
 ```
 unp4k.exe C:\Program Files\Star Citizen\LIVE\Data.p4k *.ini
-unp4k.exe C:\Program Files\Star Citizen\LIVE\Data.p4k *.xml
+unp4k.exe C:\Program Files\Star Citizen\LIVE\Data.p4k Data\Libs\Config
 ```
 That will create folders `Data` and `Engine` in the current directory.
 
-Now we have to unpack compressed xml data.
+OPTIONAL:\
+The XML files is encoded using CryEngine XMLB packing, 
+so if you want to read the data by eyes or use ot with some other tools,
+you can unpack all files. But this step is fully optional, 
+as the tool can work with packed files.
 ```
 unforge.exe Data\Libs\Config\defaultProfile.xml
 unforge.exe Data\Libs\Config\keybinding_localization.xml 
@@ -30,7 +34,7 @@ unforge.exe Data\Libs\Config\keybinding_localization.xml
 sc-keybind-extract --profile Data\Libs\Config\defaultProfile.xml --localization Data\Localization\english\global.ini > kbd.csv
 ```
 
-Additionally, you can add information about keymap changes comparing to previous version of the game.
+Additionally, you can add information about keymap changes comparing to a previous version of the game.
 To do that you have to provide the previous version of the game data as follows:
 
 ```

go.mod

@@ -3,3 +3,5 @@ module github.com/GlebYaltchik/sc-keybind-extract
 go 1.22.3
 
 require github.com/spf13/pflag v1.0.5
+
+require github.com/beevik/etree v1.4.0

go.sum

@@ -1,2 +1,4 @@
+github.com/beevik/etree v1.4.0 h1:oz1UedHRepuY3p4N5OjE0nK1WLCqtzHf25bxplKOHLs=
+github.com/beevik/etree v1.4.0/go.mod h1:cyWiXwGoasx60gHvtnEh5x8+uIjUVnjWqBvEnhnqKDA=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=

internal/cry/xmlb/parser.go

@@ -0,0 +1,155 @@
+package xmlb
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/GlebYaltchik/sc-keybind-extract/internal/stream"
+)
+
+type Node struct {
+	NodeNameOffset      int32
+	ContentOffset       int32
+	AttributeCount      int16
+	ChildCount          int16
+	ParentNodeID        int32
+	FirstAttributeIndex int32
+	FirstChildIndex     int32
+	Reserved            int32
+}
+
+type Reference struct {
+	NameOffset  int32
+	ValueOffset int32
+}
+
+type DataMap map[int32]string
+
+type info struct {
+	NodeTableOffset      int32
+	NodeTableCount       int32
+	AttributeTableOffset int32
+	AttributeTableCount  int32
+	ChildTableOffset     int32
+	ChildTableCount      int32
+	StringTableOffset    int32
+	StringTableCount     int32
+}
+
+type Parser struct {
+	s     *stream.Stream
+	info  info
+	names DataMap
+	attrs []Reference
+}
+
+func NewParser(s *stream.Stream) (*Parser, error) {
+	if s.PeekChar() != 'C' {
+		return nil, fmt.Errorf("unknown data format")
+	}
+
+	hdr := s.ReadFString(7)
+
+	switch hdr {
+	case "CryXmlB", "CryXml":
+		_ = s.ReadCString()
+	case "CRY3SDK":
+		_, _ = s.ReadByte()
+		_, _ = s.ReadByte()
+	default:
+		return nil, fmt.Errorf("unknown data format")
+	}
+
+	headerPos := s.Pos()
+
+	fileLength := s.ReadInt32()
+	if int64(fileLength) != s.Size() {
+		s.SetOrder(binary.LittleEndian)
+		_, _ = s.Seek(headerPos, io.SeekStart)
+		fileLength = s.ReadInt32()
+	}
+
+	if int64(fileLength) != s.Size() {
+		return nil, fmt.Errorf("file length mismatch")
+	}
+
+	info := mustRead[info](s)
+
+	return &Parser{
+		s:     s,
+		info:  info,
+		names: readDict(s, int64(info.StringTableOffset)),
+		attrs: readAttrs(s, int64(info.AttributeTableOffset), int(info.AttributeTableCount)),
+	}, nil
+}
+
+func (p *Parser) ReadNodes() []Node {
+	_, _ = p.s.Seek(int64(p.info.NodeTableOffset), io.SeekStart)
+
+	nodes := make([]Node, p.info.NodeTableCount)
+
+	must(p.s.ReadObject(&nodes))
+
+	return nodes
+}
+
+func (p *Parser) NodeName(n Node) string {
+	return p.names[n.NodeNameOffset]
+}
+
+func (p *Parser) NodeContent(n Node) string {
+	content, ok := p.names[n.ContentOffset]
+	if !ok {
+		content = "BUGGED"
+	}
+
+	return content
+}
+
+func (p *Parser) GetAttr(id int32) (name, value string) {
+	value, ok := p.names[p.attrs[id].ValueOffset]
+	if !ok {
+		value = "BUGGED"
+	}
+
+	name = p.names[p.attrs[id].NameOffset]
+
+	return name, value
+}
+
+func readDict(s *stream.Stream, offset int64) DataMap {
+	_, _ = s.Seek(offset, io.SeekStart)
+
+	dataMap := make(DataMap)
+
+	for s.Pos() < s.Size() {
+		dataMap[int32(s.Pos()-offset)] = s.ReadCString()
+	}
+
+	return dataMap
+}
+
+func readAttrs(s *stream.Stream, offset int64, count int) []Reference {
+	_, _ = s.Seek(offset, io.SeekStart)
+
+	attrs := make([]Reference, count)
+
+	must(s.ReadObject(&attrs))
+
+	return attrs
+}
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustRead[T any](s *stream.Stream) T {
+	var v T
+
+	must(s.ReadObject(&v))
+
+	return v
+}

internal/cry/xmlb/xmlb.go

@@ -0,0 +1,55 @@
+package xmlb
+
+import (
+	"github.com/beevik/etree"
+
+	"github.com/GlebYaltchik/sc-keybind-extract/internal/stream"
+)
+
+// This code is a porting of the CryEngine XMLB parser to Go.
+// The original code is written in C# and can be found here:
+// https://github.com/dolkensp/unp4k/blob/develop/src/unforge/CryXmlB/CryXmlSerializer.cs
+
+func Decode(data []byte) ([]byte, error) {
+	if len(data) > 0 && data[0] == '<' {
+		return data, nil // not encoded
+	}
+
+	br := stream.New(data)
+
+	p, err := NewParser(br)
+	if err != nil {
+		return nil, err
+	}
+
+	doc := etree.NewDocument()
+
+	attributeIndex := int32(0)
+	xmlMap := make(map[int]*etree.Element)
+
+	for nodeID, node := range p.ReadNodes() {
+		element := etree.NewElement(p.NodeName(node))
+
+		for i := int16(0); i < node.AttributeCount; i++ {
+			element.CreateAttr(p.GetAttr(attributeIndex))
+			attributeIndex++
+		}
+
+		xmlMap[nodeID] = element
+
+		if content := p.NodeContent(node); content != "" {
+			element.AddChild(etree.NewCData(content))
+		}
+
+		parent, ok := xmlMap[int(node.ParentNodeID)]
+		if ok {
+			parent.AddChild(element)
+		} else {
+			doc.AddChild(element)
+		}
+	}
+
+	doc.Indent(2)
+
+	return doc.WriteToBytes()
+}

internal/profile/profile.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"strings"
+
+	"github.com/GlebYaltchik/sc-keybind-extract/internal/cry/xmlb"
 )
 
 type info struct {
@@ -79,5 +81,10 @@ func DecodeFile(name string) (Actions, error) {
 		return nil, fmt.Errorf("can't read profile: %w", err)
 	}
 
+	data, err = xmlb.Decode(data)
+	if err != nil {
+		return nil, fmt.Errorf("can't decode profile: %w", err)
+	}
+
 	return Decode(data)
 }

internal/stream/stream.go

@@ -0,0 +1,95 @@
+package stream
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+)
+
+func New(data []byte) *Stream {
+	return &Stream{
+		Reader: bytes.NewReader(data),
+		order:  binary.LittleEndian,
+	}
+}
+
+type Stream struct {
+	order binary.ByteOrder
+	*bytes.Reader
+}
+
+func (s *Stream) PeekChar() byte {
+	c, _ := s.ReadByte()
+	_ = s.UnreadByte()
+
+	return c
+}
+
+func (s *Stream) ReadFString(n int) string {
+	data := make([]byte, n)
+
+	_, _ = s.Read(data)
+
+	for i := range data {
+		if data[i] == 0 {
+			return string(data[:i])
+		}
+	}
+
+	return string(data)
+}
+
+func (s *Stream) ReadCString() string {
+	start := s.Pos()
+
+	for {
+		c, err := s.ReadByte()
+		if c == 0 || err != nil {
+			break
+		}
+	}
+
+	strLen := s.Pos() - start
+
+	_, _ = s.Seek(start, io.SeekStart)
+
+	data := make([]byte, strLen)
+	n, _ := s.Read(data)
+
+	data = data[:n]
+
+	if len(data) > 0 && data[len(data)-1] == 0 {
+		data = data[:len(data)-1]
+	}
+
+	return string(data)
+}
+
+func (s *Stream) Pos() int64 {
+	pos, _ := s.Seek(0, io.SeekCurrent)
+	return pos
+}
+
+func (s *Stream) ReadInt16() int {
+	var v int16
+
+	_ = binary.Read(s, s.order, &v)
+
+	return int(v)
+}
+
+func (s *Stream) ReadInt32() int {
+	var v int32
+
+	_ = binary.Read(s, s.order, &v)
+
+	return int(v)
+}
+
+func (s *Stream) ReadObject(v any) error {
+	return binary.Read(s, s.order, v)
+}
+
+func (s *Stream) SetOrder(order binary.ByteOrder) {
+	s.order = order
+}