diff --git a/common/batch/blob.go b/common/batch/blob.go index 088e55a6f..2b3c465bb 100644 --- a/common/batch/blob.go +++ b/common/batch/blob.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - "morph-l2/node/zstd" + "morph-l2/common/codec/zstd" eth "github.com/morph-l2/go-ethereum/core/types" "github.com/morph-l2/go-ethereum/crypto/kzg4844" diff --git a/common/blob/payload.go b/common/blob/payload.go index 1b0485651..e633039a8 100644 --- a/common/blob/payload.go +++ b/common/blob/payload.go @@ -3,7 +3,7 @@ package blob import ( "fmt" - "morph-l2/node/zstd" + "morph-l2/common/codec/zstd" "github.com/morph-l2/go-ethereum/common" eth "github.com/morph-l2/go-ethereum/core/types" diff --git a/common/codec/zstd/codec.go b/common/codec/zstd/codec.go new file mode 100644 index 000000000..3fd6fdc28 --- /dev/null +++ b/common/codec/zstd/codec.go @@ -0,0 +1,15 @@ +package zstd + +import codeczstd "github.com/morph-l2/morph-da-codec/bindings/codec/zstd" + +func CompressBatchBytes(batchBytes []byte) ([]byte, error) { + if len(batchBytes) == 0 { + return nil, nil + } + return codeczstd.CompressMorphDABatch(batchBytes) +} + +// DecompressBatchBytes decompresses the given bytes into batch bytes +func DecompressBatchBytes(compressedBytes []byte) ([]byte, error) { + return codeczstd.DecompressMorphDABatch(compressedBytes) +} diff --git a/common/codec/zstd/codec_test.go b/common/codec/zstd/codec_test.go new file mode 100644 index 000000000..d09274a00 --- /dev/null +++ b/common/codec/zstd/codec_test.go @@ -0,0 +1,42 @@ +package zstd + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" +) + +var transactions = [][]byte{ + mustDecodeHex("f8ac820c7d84a0eebb0082d501949e12ad42c4e4d2acfbade01a96446e48e6764b9880b844a9059cbb00000000000000000000000071e415b922dfea87cca82e1a33412ae5f750746300000000000000000000000000000000000000000000000000038d7ea4c68000821617a0086177091aad7d0e292a4abbe54de5ef36b0730c329bb30b00505068ebc2bfeca019cae91f3f9a33574e1fcda8497c0df27f922b11b6a34a27c446811cb77cd163"), + mustDecodeHex("f8d281e284773594008304777394530000000000000000000000000000000000000286ea31e4079800b8642fcc29fa0000000000000000000000000c178d8d1e6facfc5938ff36d629ff7ebf6a6f970000000000000000000000000000000000000000000000000000ea31e40798000000000000000000000000000000000000000000000000000000000000047773821617a01a6854ce0d0cd5e9a83d373a6b09ff163d462d012726686af8bee9a3c3f68d3fa035a1b70746de540c059684dd4097a0332afe453cd833e52d745765adee071e80"), + mustDecodeHex("f9010e830812a784773594008301819294b17be239cf3c15b33cb865d4ace5e28aa883440b80b8a456d30bf8000000000000000000000000dd6e11df53ce84811844c4233b0807a2e95c61d80000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333538000000000000821618a02ed3c3e996eedf2485f46d285c0fc8680c6a0087f06f059b8e4b73f7f79b6bcaa0077a7142e81a8ed6c5a84462d1ff24cd6a7f5d8f65074145ca9957b38ca51b9d"), + mustDecodeHex("f9010e830812a884773594008301819294b17be239cf3c15b33cb865d4ace5e28aa883440b80b8a456d30bf80000000000000000000000004e2cb6e329cc6ea1e17abf1a15a2e148f55c1cfb0000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333630000000000000821618a0da7efb4c6b883f163955acd343abbf4882c94b102883d86ca613c688bca5e7f9a009a2d23644ed53a403d139d0b986b4bee3738c28a810dc6b4295d82fc54e4825"), + mustDecodeHex("f8a926847735940082f574947d9f7399951c96c83df20c4839cfcd1e79c9d7f680b8446e553f650000000000000000000000000000000000000000000000004563918244f40000000000000000000000000000dc3f2c5a61053073999548dc3f8372082c4b6a1f8216189f9d5ae2401b6e567243afd59dc44490e6d38cbaefa75aad3fec5d0ba608b9c9a01702d263ccce7bc6661d89147d7a1b31dc7a501868124ccabd01633ac281f657"), + mustDecodeHex("f9010e83079d78847735940083018192940d763880cc7e54749e4fe3065db53da839a8ef6b80b8a456d30bf8000000000000000000000000245e1f5fd0b30ef3bf1a18f5718621b49ad9fd660000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333632000000000000821617a0d6c343f6c301d3220c528923404826369b04c35afb9e6f33a86d766d47086391a00d50ca672f329b5c63da325d77f288dce41e1aa0c2ee381095a282bfbb07139f"), + mustDecodeHex("f8aa30847735940082e1ec94aebc89aff5ad69d7cf8b85c54d394ad34d9c46bb80b8446e553f65000000000000000000000000000000000000000000000004e1003b28d928000000000000000000000000000044e0c443d145ffce67167550b219ed2728b2f972821617a0aeed20bef6379d2be898920d6d9028bcdc23f39198a655ba33f20202c4483551a07f45d819f57d9a7d92201628739e0d82124e238b7ef52a88cdc5cd94dfc1ea2f"), + mustDecodeHex("f9010e83079d79847735940083018192940d763880cc7e54749e4fe3065db53da839a8ef6b80b8a456d30bf800000000000000000000000043634f04587cc3617621661444113899e2f5bba40000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333634000000000000821617a0f535fd6a8ab7136f7944aaa217a7e253514034e164daa43abcd6f8849b240263a03ebe9adf23f63239a60c5a4b60857b0620e62f8826f2e6564adac9839f1814c4"), + mustDecodeHex("f905358206b28477359400830474f9949f6e8e1c33fc2aa6ff29b747922e3f8e32911b9d87071afd498d0000b904c4c7cd97480000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000eccaeb7e3928900100c89809db5682a835902900000000000000000000000000000000000000000000000000071afd498d000000000000000000000000000000000000000000000000000000650bb12ba9250600000000000000000000000067297ee4eb097e072b4ab6f1620268061ae8046400000000000000000000000060cba82ddbf4b5ddcd4398cdd05354c6a790c309000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000032000000000000000000000000000000000000000000000000000000000000001490000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007968156adf895922406523887e5c157623c38db00eccaeb7e3928900100c89809db5682a83590299e12ad42c4e4d2acfbade01a96446e48e6764b98530000000000000000000000000000000000001100000000000000000000000000000000000000000000000000513e24f614378000000000000000000000000000000000000000000000000000071afd498d0000000000000000000000000000000000000000000000000000000000006671399d00eccaeb7e3928900100c89809db5682a83590291a9d3116643841c08afc92c883e800af00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004126638c4498d277a287f07d8836fd0d54aa2048958af0bfe3f82fa31dc35aefc27ffff753faa3d924f4655089d42a772b89b1a1d22838e5e624546addc3d31b8e1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000010438ed173900000000000000000000000000000000000000000000000000071afd498d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000eccaeb7e3928900100c89809db5682a835902900000000000000000000000000000000000000000000000000000000667113b9000000000000000000000000000000000000000000000000000000000000000200000000000000000000000053000000000000000000000000000000000000110000000000000000000000009e12ad42c4e4d2acfbade01a96446e48e6764b9800000000000000000000000000000000000000000000000000000000821617a0d572c2930049defb041e2900976a31a5b4223f9e81b2bf11a754be95acbb8623a00324cdd6685bd19812ba82cefab7c2cd4d65385b61d6bdb8a017e4ddbf841bcd"), + mustDecodeHex("f88a08847735940083044dfe9437088ecc92aa376f1535bd42186efc3a92b39e6580a40e7527020000000000000000000000000000000000000000000000000000000000000030821618a0fe81af4d4cad761dba85dd413a78ecfdb6242c05003980b1a3046eb5fdd762b9a0712e23aa0152de5259365299fa992e99d84336e10ce79eeb841e9b5038964877"), +} + +func mustDecodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func TestCompressDecompress(t *testing.T) { + var input []byte + for _, txBz := range transactions { + input = append(input, txBz...) + } + output, err := CompressBatchBytes(input) + require.NoError(t, err) + + decompressed, err := DecompressBatchBytes(output) + require.NoError(t, err) + require.EqualValues(t, input, decompressed) +} diff --git a/common/go.mod b/common/go.mod index 099602c1c..5c45590c2 100644 --- a/common/go.mod +++ b/common/go.mod @@ -6,6 +6,7 @@ replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3. require ( github.com/holiman/uint256 v1.2.4 + github.com/morph-l2/morph-da-codec/bindings/codec v0.0.0-20260702164327-8c7baa26af85 github.com/morph-l2/go-ethereum v1.10.14-0.20260709072051-ffa95e0670a3 github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a diff --git a/common/go.sum b/common/go.sum index 61926eb67..837361a8c 100644 --- a/common/go.sum +++ b/common/go.sum @@ -148,6 +148,8 @@ github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqky github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morph-l2/morph-da-codec/bindings/codec v0.0.0-20260702164327-8c7baa26af85 h1:987LsP4lErsu3D/9LKMtqhuU2k3ip9lp2IIMGCmXhRE= +github.com/morph-l2/morph-da-codec/bindings/codec v0.0.0-20260702164327-8c7baa26af85/go.mod h1:f6+BHrrHbRmEYplC1nmNjGdatQtMSa2CIK7CX8T1Nfc= github.com/morph-l2/go-ethereum v1.10.14-0.20260709072051-ffa95e0670a3 h1:vt1tCsMO0eFZQsNnI3KRk76V52ia19Z/Zbpk92x0w3Y= github.com/morph-l2/go-ethereum v1.10.14-0.20260709072051-ffa95e0670a3/go.mod h1:nkVzHjQWCOjvukQW8ittlwX+Xz9gmVHrP7mUi7zoHTs= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/go.work.sum b/go.work.sum index acba38e6c..feb69c4bf 100644 --- a/go.work.sum +++ b/go.work.sum @@ -993,6 +993,7 @@ github.com/morph-l2/go-ethereum v0.0.0-20260529141627-eb5fbf8f9748/go.mod h1:nkV github.com/morph-l2/go-ethereum v1.10.14-0.20251125061742-69718a9dcab9/go.mod h1:tiFPeidxjoCmLj18ne9H3KQdIGTCvRC30qlef06Fd9M= github.com/morph-l2/tendermint v0.0.0-20260529095305-b1b3a3a1d806 h1:gJmofzJ0PnCCObETUH02AsSVS1YY4tLpUWYFhEggCmk= github.com/morph-l2/tendermint v0.0.0-20260529095305-b1b3a3a1d806/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= +github.com/morph-l2/tendermint v0.3.4/go.mod h1:TtCzp9l6Z6yDUiwv3TbqKqw8Q8RKp3fSz5+adO1/Y8w= github.com/morph-l2/tendermint v0.3.8-0.20260617072029-29056623cdb0 h1:LcRbLo0pQXev4z/6i65mVr/9XoFE9Zf7+tcqKItpE+M= github.com/morph-l2/tendermint v0.3.8-0.20260617072029-29056623cdb0/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= diff --git a/node/derivation/batch_info.go b/node/derivation/batch_info.go index 020c13acf..05f0779dc 100644 --- a/node/derivation/batch_info.go +++ b/node/derivation/batch_info.go @@ -13,8 +13,8 @@ import ( "github.com/morph-l2/go-ethereum/eth/catalyst" commonbatch "morph-l2/common/batch" + "morph-l2/common/codec/zstd" "morph-l2/node/types" - "morph-l2/node/zstd" ) type BlockContext struct { diff --git a/node/derivation/batch_info_test.go b/node/derivation/batch_info_test.go index e3f1852ce..38e9eec82 100644 --- a/node/derivation/batch_info_test.go +++ b/node/derivation/batch_info_test.go @@ -14,8 +14,8 @@ import ( commonbatch "morph-l2/common/batch" "morph-l2/common/blob" + "morph-l2/common/codec/zstd" "morph-l2/node/types" - "morph-l2/node/zstd" ) // buildBlockContexts returns the concatenated 60-byte encoding of `count` @@ -72,6 +72,36 @@ func splitCompressedIntoBlobs(t *testing.T, compressed []byte) []kzg4844.Blob { return blobs } +func appendRandomTrailer(t *testing.T, data []byte, trailerLen int) []byte { + t.Helper() + trailer := make([]byte, trailerLen) + _, err := rand.Read(trailer) + require.NoError(t, err) + if trailerLen > 0 { + // Ensure the suffix is visibly invalid tx data if a decoder ever + // leaks it into the decompressed payload. + trailer[0] = 0x05 + } + out := make([]byte, 0, len(data)+trailerLen) + out = append(out, data...) + out = append(out, trailer...) + return out +} + +func buildRandomBatchPayload(t *testing.T, startBlock uint64, blockCount, padLen int) []byte { + t.Helper() + blockCtx := buildBlockContexts(startBlock, blockCount) + pad := make([]byte, padLen) + _, err := rand.Read(pad) + require.NoError(t, err) + + payload := make([]byte, 0, len(blockCtx)+1+padLen) + payload = append(payload, blockCtx...) + payload = append(payload, 0x00) + payload = append(payload, pad...) + return payload +} + // TestParseBatchSingleBlob covers the backward-compatible path where a V1 // batch fits in a single blob. It guards against regressions in the recent // "concatenate then decompress" refactor: the single-blob flow must still @@ -226,6 +256,136 @@ func TestParseBatchMultiBlobConcatDecompressInvariant(t *testing.T) { } } +// TestParseBatchIgnoresTrailingBytesAfterZstdFrame guards the zstd decoder +// differential fixed in bug #10. The batch payload is a single valid zstd +// frame; arbitrary bytes appended after that frame must not become part of +// the decompressed tx stream, otherwise DecodeTxsFromBytes may reject the +// immutable L1 batch forever. Cover both layouts the derivation code sees: +// the whole encoded stream in one blob and the same stream split over +// multiple blobs. +func TestParseBatchIgnoresTrailingBytesAfterZstdFrame(t *testing.T) { + const ( + parentIndex = 321 + startBlock = 3_000 + blockCount = 6 + trailerLen = 384 + ) + + tests := []struct { + name string + version uint + padLen int + assertLayout func(*testing.T, []byte, []kzg4844.Blob) + }{ + { + name: "single blob", + version: 1, + padLen: 512, + assertLayout: func(t *testing.T, compressed []byte, blobs []kzg4844.Blob) { + t.Helper() + require.LessOrEqual(t, len(compressed), commonbatch.MaxBlobBytesSize, + "single-blob test expects compressed payload plus trailer to fit in one blob") + require.Len(t, blobs, 1) + }, + }, + { + name: "multi blob", + version: 2, + padLen: commonbatch.MaxBlobBytesSize + commonbatch.MaxBlobBytesSize/5, + assertLayout: func(t *testing.T, compressed []byte, blobs []kzg4844.Blob) { + t.Helper() + require.Greater(t, len(compressed), commonbatch.MaxBlobBytesSize, + "multi-blob test requires compressed payload plus trailer to overflow one blob") + require.GreaterOrEqual(t, len(blobs), 2) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload := buildRandomBatchPayload(t, startBlock, blockCount, tt.padLen) + + compressed, err := zstd.CompressBatchBytes(payload) + require.NoError(t, err) + compressedWithTrailer := appendRandomTrailer(t, compressed, trailerLen) + + decoded, err := zstd.DecompressBatchBytes(compressedWithTrailer) + require.NoError(t, err) + require.Equal(t, payload, decoded) + + blobs := splitCompressedIntoBlobs(t, compressedWithTrailer) + tt.assertLayout(t, compressedWithTrailer, blobs) + + batch := geth.RPCRollupBatch{ + Version: tt.version, + ParentBatchHeader: buildV1ParentHeader(parentIndex, startBlock), + LastBlockNumber: startBlock + blockCount - 1, + PrevStateRoot: common.BigToHash(big.NewInt(1)), + PostStateRoot: common.BigToHash(big.NewInt(2)), + WithdrawRoot: common.BigToHash(big.NewInt(3)), + Sidecar: eth.BlobTxSidecar{Blobs: blobs}, + } + + var bi BatchInfo + require.NoError(t, bi.ParseBatch(batch)) + require.EqualValues(t, parentIndex+1, bi.batchIndex) + require.EqualValues(t, startBlock, bi.FirstBlockNumber()) + require.EqualValues(t, startBlock+blockCount-1, bi.LastBlockNumber()) + require.Len(t, bi.blockContexts, blockCount) + }) + } +} + +// TestParseBatchIgnoresConcatenatedZstdFrame covers the exploit shape behind +// bug #10: a valid second frame must be ignored to match the zkVM's +// single-frame decoder. The first frame intentionally ends immediately after +// the block contexts. If the second frame is decoded, its leading unsupported +// tx type (0x05) makes ParseBatch fail and exposes the differential. +func TestParseBatchIgnoresConcatenatedZstdFrame(t *testing.T) { + const ( + parentIndex = 322 + startBlock = 4_000 + blockCount = 4 + trailerLen = 384 + ) + + payload := buildBlockContexts(startBlock, blockCount) + compressed, err := zstd.CompressBatchBytes(payload) + require.NoError(t, err) + + trailingPayload := make([]byte, trailerLen) + _, err = rand.Read(trailingPayload) + require.NoError(t, err) + trailingPayload[0] = 0x05 + trailingFrame, err := zstd.CompressBatchBytes(trailingPayload) + require.NoError(t, err) + + concatenatedFrames := make([]byte, 0, len(compressed)+len(trailingFrame)) + concatenatedFrames = append(concatenatedFrames, compressed...) + concatenatedFrames = append(concatenatedFrames, trailingFrame...) + + decoded, err := zstd.DecompressBatchBytes(concatenatedFrames) + require.NoError(t, err) + require.Equal(t, payload, decoded) + + blobs := splitCompressedIntoBlobs(t, concatenatedFrames) + require.Len(t, blobs, 1) + + batch := geth.RPCRollupBatch{ + Version: 1, + ParentBatchHeader: buildV1ParentHeader(parentIndex, startBlock), + LastBlockNumber: startBlock + blockCount - 1, + Sidecar: eth.BlobTxSidecar{Blobs: blobs}, + } + + var bi BatchInfo + require.NoError(t, bi.ParseBatch(batch)) + require.EqualValues(t, parentIndex+1, bi.batchIndex) + require.EqualValues(t, startBlock, bi.FirstBlockNumber()) + require.EqualValues(t, startBlock+blockCount-1, bi.LastBlockNumber()) + require.Len(t, bi.blockContexts, blockCount) +} + // TestParseBatchBlockCountBounds covers the blockCount guards hardened in // issue #994, both of which would otherwise panic and crash layer1-verify // nodes. A zero count (lastBlockNumber == parent) yielded empty blockContexts diff --git a/node/go.mod b/node/go.mod index 5726149d3..226c37abf 100644 --- a/node/go.mod +++ b/node/go.mod @@ -12,7 +12,6 @@ require ( github.com/hashicorp/golang-lru v1.0.2 github.com/hashicorp/raft v1.7.3 github.com/hashicorp/raft-boltdb/v2 v2.3.1 - github.com/klauspost/compress v1.17.9 github.com/mdlayher/vsock v1.2.1 github.com/morph-l2/go-ethereum v1.10.14-0.20260709072051-ffa95e0670a3 github.com/pkg/errors v0.9.1 @@ -80,6 +79,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmhodges/levigo v1.0.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/magiconair/properties v1.8.6 // indirect diff --git a/node/zstd/codec.go b/node/zstd/codec.go deleted file mode 100644 index 6bb1866e5..000000000 --- a/node/zstd/codec.go +++ /dev/null @@ -1,62 +0,0 @@ -package zstd - -/* -#include -char* compress_scroll_batch_bytes(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size); -*/ -import "C" - -import ( - "bytes" - "fmt" - "unsafe" - - "github.com/klauspost/compress/zstd" -) - -var magics = []byte{0x28, 0xb5, 0x2f, 0xfd} - -func CompressBatchBytes(bytes []byte) ([]byte, error) { - srcSize := C.uint64_t(len(bytes)) - outbufSize := C.uint64_t(len(bytes) + 128) // Allocate output buffer with extra 128 bytes - outbuf := make([]byte, outbufSize) - - if err := C.compress_scroll_batch_bytes((*C.uchar)(unsafe.Pointer(&bytes[0])), srcSize, - (*C.uchar)(unsafe.Pointer(&outbuf[0])), &outbufSize); err != nil { - return nil, fmt.Errorf("failed to compress batch bytes: %s", C.GoString(err)) - } - - return outbuf[:int(outbufSize)], nil -} - -// DecompressBatchBytes decompresses the given bytes into batch bytes -func DecompressBatchBytes(compressedBytes []byte) ([]byte, error) { - // add magics - data := make([]byte, len(compressedBytes)+len(magics)) - copy(data, magics) - copy(data[len(magics):], compressedBytes) - - // decompress data in stream and in batches of bytes, because we don't know actual length of compressed data - var res []byte - readBatchSize := 131072 - batchOfBytes := make([]byte, readBatchSize) - - r := bytes.NewReader(data) - zr, err := zstd.NewReader(r) - if err != nil { - return nil, err - } - defer zr.Close() - - for { - i, _ := zr.Read(batchOfBytes) // - res = append(res, batchOfBytes[:i]...) - if i < readBatchSize { - break - } - } - if len(res) == 0 { - return nil, fmt.Errorf("failed to decompress blob bytes") - } - return res, nil -} diff --git a/node/zstd/codec_test.go b/node/zstd/codec_test.go deleted file mode 100644 index f16f847f4..000000000 --- a/node/zstd/codec_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package zstd - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/ethutil/hex" -) - -var transactions = [][]byte{ - hex.MustDecodeHex("f8ac820c7d84a0eebb0082d501949e12ad42c4e4d2acfbade01a96446e48e6764b9880b844a9059cbb00000000000000000000000071e415b922dfea87cca82e1a33412ae5f750746300000000000000000000000000000000000000000000000000038d7ea4c68000821617a0086177091aad7d0e292a4abbe54de5ef36b0730c329bb30b00505068ebc2bfeca019cae91f3f9a33574e1fcda8497c0df27f922b11b6a34a27c446811cb77cd163"), - hex.MustDecodeHex("f8d281e284773594008304777394530000000000000000000000000000000000000286ea31e4079800b8642fcc29fa0000000000000000000000000c178d8d1e6facfc5938ff36d629ff7ebf6a6f970000000000000000000000000000000000000000000000000000ea31e40798000000000000000000000000000000000000000000000000000000000000047773821617a01a6854ce0d0cd5e9a83d373a6b09ff163d462d012726686af8bee9a3c3f68d3fa035a1b70746de540c059684dd4097a0332afe453cd833e52d745765adee071e80"), - hex.MustDecodeHex("f9010e830812a784773594008301819294b17be239cf3c15b33cb865d4ace5e28aa883440b80b8a456d30bf8000000000000000000000000dd6e11df53ce84811844c4233b0807a2e95c61d80000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333538000000000000821618a02ed3c3e996eedf2485f46d285c0fc8680c6a0087f06f059b8e4b73f7f79b6bcaa0077a7142e81a8ed6c5a84462d1ff24cd6a7f5d8f65074145ca9957b38ca51b9d"), - hex.MustDecodeHex("f9010e830812a884773594008301819294b17be239cf3c15b33cb865d4ace5e28aa883440b80b8a456d30bf80000000000000000000000004e2cb6e329cc6ea1e17abf1a15a2e148f55c1cfb0000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333630000000000000821618a0da7efb4c6b883f163955acd343abbf4882c94b102883d86ca613c688bca5e7f9a009a2d23644ed53a403d139d0b986b4bee3738c28a810dc6b4295d82fc54e4825"), - hex.MustDecodeHex("f8a926847735940082f574947d9f7399951c96c83df20c4839cfcd1e79c9d7f680b8446e553f650000000000000000000000000000000000000000000000004563918244f40000000000000000000000000000dc3f2c5a61053073999548dc3f8372082c4b6a1f8216189f9d5ae2401b6e567243afd59dc44490e6d38cbaefa75aad3fec5d0ba608b9c9a01702d263ccce7bc6661d89147d7a1b31dc7a501868124ccabd01633ac281f657"), - hex.MustDecodeHex("f9010e83079d78847735940083018192940d763880cc7e54749e4fe3065db53da839a8ef6b80b8a456d30bf8000000000000000000000000245e1f5fd0b30ef3bf1a18f5718621b49ad9fd660000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333632000000000000821617a0d6c343f6c301d3220c528923404826369b04c35afb9e6f33a86d766d47086391a00d50ca672f329b5c63da325d77f288dce41e1aa0c2ee381095a282bfbb07139f"), - hex.MustDecodeHex("f8aa30847735940082e1ec94aebc89aff5ad69d7cf8b85c54d394ad34d9c46bb80b8446e553f65000000000000000000000000000000000000000000000004e1003b28d928000000000000000000000000000044e0c443d145ffce67167550b219ed2728b2f972821617a0aeed20bef6379d2be898920d6d9028bcdc23f39198a655ba33f20202c4483551a07f45d819f57d9a7d92201628739e0d82124e238b7ef52a88cdc5cd94dfc1ea2f"), - hex.MustDecodeHex("f9010e83079d79847735940083018192940d763880cc7e54749e4fe3065db53da839a8ef6b80b8a456d30bf800000000000000000000000043634f04587cc3617621661444113899e2f5bba40000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000001a3332343337323839373334373839303133313338353731333634000000000000821617a0f535fd6a8ab7136f7944aaa217a7e253514034e164daa43abcd6f8849b240263a03ebe9adf23f63239a60c5a4b60857b0620e62f8826f2e6564adac9839f1814c4"), - hex.MustDecodeHex("f905358206b28477359400830474f9949f6e8e1c33fc2aa6ff29b747922e3f8e32911b9d87071afd498d0000b904c4c7cd97480000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000eccaeb7e3928900100c89809db5682a835902900000000000000000000000000000000000000000000000000071afd498d000000000000000000000000000000000000000000000000000000650bb12ba9250600000000000000000000000067297ee4eb097e072b4ab6f1620268061ae8046400000000000000000000000060cba82ddbf4b5ddcd4398cdd05354c6a790c309000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000032000000000000000000000000000000000000000000000000000000000000001490000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007968156adf895922406523887e5c157623c38db00eccaeb7e3928900100c89809db5682a83590299e12ad42c4e4d2acfbade01a96446e48e6764b98530000000000000000000000000000000000001100000000000000000000000000000000000000000000000000513e24f614378000000000000000000000000000000000000000000000000000071afd498d0000000000000000000000000000000000000000000000000000000000006671399d00eccaeb7e3928900100c89809db5682a83590291a9d3116643841c08afc92c883e800af00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004126638c4498d277a287f07d8836fd0d54aa2048958af0bfe3f82fa31dc35aefc27ffff753faa3d924f4655089d42a772b89b1a1d22838e5e624546addc3d31b8e1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000010438ed173900000000000000000000000000000000000000000000000000071afd498d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000eccaeb7e3928900100c89809db5682a835902900000000000000000000000000000000000000000000000000000000667113b9000000000000000000000000000000000000000000000000000000000000000200000000000000000000000053000000000000000000000000000000000000110000000000000000000000009e12ad42c4e4d2acfbade01a96446e48e6764b9800000000000000000000000000000000000000000000000000000000821617a0d572c2930049defb041e2900976a31a5b4223f9e81b2bf11a754be95acbb8623a00324cdd6685bd19812ba82cefab7c2cd4d65385b61d6bdb8a017e4ddbf841bcd"), - hex.MustDecodeHex("f88a08847735940083044dfe9437088ecc92aa376f1535bd42186efc3a92b39e6580a40e7527020000000000000000000000000000000000000000000000000000000000000030821618a0fe81af4d4cad761dba85dd413a78ecfdb6242c05003980b1a3046eb5fdd762b9a0712e23aa0152de5259365299fa992e99d84336e10ce79eeb841e9b5038964877"), -} - -func TestCompressDecompress(t *testing.T) { - var input []byte - for _, txBz := range transactions { - input = append(input, txBz...) - } - output, err := CompressBatchBytes(input) - require.NoError(t, err) - - decompressed, err := DecompressBatchBytes(output) - require.NoError(t, err) - require.EqualValues(t, input, decompressed) -} diff --git a/node/zstd/libscroll_zstd_darwin_arm64.a b/node/zstd/libscroll_zstd_darwin_arm64.a deleted file mode 100644 index 7180e1cf6..000000000 Binary files a/node/zstd/libscroll_zstd_darwin_arm64.a and /dev/null differ diff --git a/node/zstd/libscroll_zstd_darwin_arm64.go b/node/zstd/libscroll_zstd_darwin_arm64.go deleted file mode 100644 index d83ec17fd..000000000 --- a/node/zstd/libscroll_zstd_darwin_arm64.go +++ /dev/null @@ -1,6 +0,0 @@ -package zstd - -/* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_darwin_arm64.a -*/ -import "C" diff --git a/node/zstd/libscroll_zstd_linux_amd64.a b/node/zstd/libscroll_zstd_linux_amd64.a deleted file mode 100644 index b2e6040b9..000000000 Binary files a/node/zstd/libscroll_zstd_linux_amd64.a and /dev/null differ diff --git a/node/zstd/libscroll_zstd_linux_amd64.go b/node/zstd/libscroll_zstd_linux_amd64.go deleted file mode 100644 index f1a686e84..000000000 --- a/node/zstd/libscroll_zstd_linux_amd64.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !musl -// +build !musl - -package zstd - -/* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_linux_amd64.a -*/ -import "C" diff --git a/node/zstd/libscroll_zstd_linux_arm64.a b/node/zstd/libscroll_zstd_linux_arm64.a deleted file mode 100644 index 999dcd556..000000000 Binary files a/node/zstd/libscroll_zstd_linux_arm64.a and /dev/null differ diff --git a/node/zstd/libscroll_zstd_linux_arm64.go b/node/zstd/libscroll_zstd_linux_arm64.go deleted file mode 100644 index f3775d23a..000000000 --- a/node/zstd/libscroll_zstd_linux_arm64.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !musl -// +build !musl - -package zstd - -/* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_linux_arm64.a -*/ -import "C"