Skip to content

Commit 4a3982e

Browse files
End-reyEnd-rey
committed
fstree: make a new object storage structure
Store objects in the format "len(Header)+len(Payload)+header+payload", for quick reading of Head, GetRange, GetStream. ``` goos: linux goarch: amd64 pkg: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics │ oldGet.txt │ newGet.txt │ │ sec/op │ sec/op vs base │ FSTree_Get/Empty/Get_regular-16 69.61µ ± 4% 68.18µ ± 4% ~ (p=0.394 n=6) FSTree_Get/Empty/Get_combined-16 86.08µ ± 4% 81.86µ ± 5% -4.90% (p=0.041 n=6) FSTree_Get/Empty/Get_compressed-16 39.61µ ± 8% 69.87µ ± 4% +76.41% (p=0.002 n=6) FSTree_Get/100B/Get_regular-16 71.42µ ± 5% 70.72µ ± 4% ~ (p=1.000 n=6) FSTree_Get/100B/Get_combined-16 86.52µ ± 5% 84.96µ ± 3% ~ (p=0.310 n=6) FSTree_Get/100B/Get_compressed-16 35.76µ ± 11% 36.24µ ± 14% ~ (p=0.937 n=6) FSTree_Get/4KB/Get_regular-16 77.91µ ± 5% 80.17µ ± 2% ~ (p=0.065 n=6) FSTree_Get/4KB/Get_combined-16 95.67µ ± 2% 95.42µ ± 2% ~ (p=0.589 n=6) FSTree_Get/4KB/Get_compressed-16 54.92µ ± 9% 62.07µ ± 6% +13.00% (p=0.009 n=6) FSTree_Get/16KB/Get_regular-16 95.83µ ± 3% 102.28µ ± 4% +6.72% (p=0.002 n=6) FSTree_Get/16KB/Get_combined-16 112.9µ ± 1% 120.4µ ± 3% +6.64% (p=0.002 n=6) FSTree_Get/16KB/Get_compressed-16 81.29µ ± 5% 89.98µ ± 2% +10.68% (p=0.002 n=6) FSTree_Get/32KB/Get_regular-16 122.9µ ± 2% 151.0µ ± 2% +22.85% (p=0.002 n=6) FSTree_Get/32KB/Get_combined-16 145.6µ ± 5% 166.9µ ± 1% +14.64% (p=0.002 n=6) FSTree_Get/32KB/Get_compressed-16 107.3µ ± 2% 119.0µ ± 2% +10.90% (p=0.002 n=6) FSTree_Get/100KB/Get_regular-16 212.7µ ± 2% 290.8µ ± 3% +36.74% (p=0.002 n=6) FSTree_Get/100KB/Get_combined-16 236.1µ ± 1% 305.6µ ± 1% +29.48% (p=0.002 n=6) FSTree_Get/100KB/Get_compressed-16 176.7µ ± 2% 201.9µ ± 2% +14.25% (p=0.002 n=6) FSTree_Get/1MB/Get_regular-16 1.509m ± 11% 2.015m ± 18% +33.54% (p=0.002 n=6) FSTree_Get/1MB/Get_combined-16 1.655m ± 6% 2.382m ± 7% +43.98% (p=0.002 n=6) FSTree_Get/1MB/Get_compressed-16 928.4µ ± 3% 1161.3µ ± 2% +25.08% (p=0.002 n=6) geomean 137.0µ 157.2µ +14.75% │ oldGet.txt │ newGet.txt │ │ B/op │ B/op vs base │ FSTree_Get/Empty/Get_regular-16 9.035Ki ± 5% 9.195Ki ± 4% ~ (p=0.260 n=6) FSTree_Get/Empty/Get_combined-16 9.122Ki ± 1% 9.149Ki ± 2% ~ (p=0.515 n=6) FSTree_Get/Empty/Get_compressed-16 10.922Ki ± 7% 9.285Ki ± 5% -14.99% (p=0.002 n=6) FSTree_Get/100B/Get_regular-16 9.391Ki ± 4% 10.957Ki ± 6% +16.68% (p=0.002 n=6) FSTree_Get/100B/Get_combined-16 9.424Ki ± 1% 11.063Ki ± 2% +17.39% (p=0.002 n=6) FSTree_Get/100B/Get_compressed-16 10.94Ki ± 7% 11.18Ki ± 6% ~ (p=0.589 n=6) FSTree_Get/4KB/Get_regular-16 17.71Ki ± 2% 23.81Ki ± 1% +34.46% (p=0.002 n=6) FSTree_Get/4KB/Get_combined-16 17.62Ki ± 1% 23.70Ki ± 0% +34.50% (p=0.002 n=6) FSTree_Get/4KB/Get_compressed-16 23.65Ki ± 1% 28.54Ki ± 1% +20.66% (p=0.002 n=6) FSTree_Get/16KB/Get_regular-16 41.64Ki ± 1% 59.77Ki ± 0% +43.54% (p=0.002 n=6) FSTree_Get/16KB/Get_combined-16 41.68Ki ± 0% 59.69Ki ± 0% +43.20% (p=0.002 n=6) FSTree_Get/16KB/Get_compressed-16 59.70Ki ± 1% 77.48Ki ± 0% +29.77% (p=0.002 n=6) FSTree_Get/32KB/Get_regular-16 79.53Ki ± 0% 119.73Ki ± 0% +50.55% (p=0.002 n=6) FSTree_Get/32KB/Get_combined-16 79.68Ki ± 0% 119.66Ki ± 0% +50.17% (p=0.002 n=6) FSTree_Get/32KB/Get_compressed-16 119.8Ki ± 0% 159.8Ki ± 0% +33.35% (p=0.002 n=6) FSTree_Get/100KB/Get_regular-16 215.9Ki ± 0% 319.8Ki ± 0% +48.15% (p=0.002 n=6) FSTree_Get/100KB/Get_combined-16 215.7Ki ± 0% 319.6Ki ± 0% +48.15% (p=0.002 n=6) FSTree_Get/100KB/Get_compressed-16 319.7Ki ± 0% 423.7Ki ± 0% +32.52% (p=0.002 n=6) FSTree_Get/1MB/Get_regular-16 2.016Mi ± 0% 3.023Mi ± 0% +49.99% (p=0.002 n=6) FSTree_Get/1MB/Get_combined-16 2.015Mi ± 0% 3.023Mi ± 0% +50.01% (p=0.002 n=6) FSTree_Get/1MB/Get_compressed-16 3.023Mi ± 0% 4.031Mi ± 0% +33.32% (p=0.002 n=6) geomean 64.48Ki 82.69Ki +28.25% │ oldGet.txt │ newGet.txt │ │ allocs/op │ allocs/op vs base │ FSTree_Get/Empty/Get_regular-16 134.5 ± 7% 136.5 ± 8% ~ (p=0.455 n=6) FSTree_Get/Empty/Get_combined-16 135.5 ± 2% 136.5 ± 3% ~ (p=0.455 n=6) FSTree_Get/Empty/Get_compressed-16 148.0 ± 15% 140.0 ± 5% ~ (p=0.156 n=6) FSTree_Get/100B/Get_regular-16 137.0 ± 6% 137.0 ± 7% ~ (p=1.000 n=6) FSTree_Get/100B/Get_combined-16 136.0 ± 1% 137.0 ± 3% +0.74% (p=0.045 n=6) FSTree_Get/100B/Get_compressed-16 137.0 ± 9% 136.0 ± 10% ~ (p=0.909 n=6) FSTree_Get/4KB/Get_regular-16 139.0 ± 6% 142.5 ± 5% ~ (p=0.290 n=6) FSTree_Get/4KB/Get_combined-16 136.5 ± 3% 138.5 ± 2% ~ (p=0.095 n=6) FSTree_Get/4KB/Get_compressed-16 137.0 ± 7% 143.5 ± 5% ~ (p=0.091 n=6) FSTree_Get/16KB/Get_regular-16 138.0 ± 7% 142.0 ± 5% ~ (p=0.481 n=6) FSTree_Get/16KB/Get_combined-16 137.0 ± 1% 138.5 ± 2% ~ (p=0.126 n=6) FSTree_Get/16KB/Get_compressed-16 139.0 ± 6% 134.0 ± 7% ~ (p=0.130 n=6) FSTree_Get/32KB/Get_regular-16 134.0 ± 8% 140.5 ± 4% +4.85% (p=0.030 n=6) FSTree_Get/32KB/Get_combined-16 137.5 ± 3% 138.0 ± 2% ~ (p=0.920 n=6) FSTree_Get/32KB/Get_compressed-16 142.0 ± 5% 142.0 ± 7% ~ (p=0.916 n=6) FSTree_Get/100KB/Get_regular-16 143.0 ± 12% 142.5 ± 8% ~ (p=0.784 n=6) FSTree_Get/100KB/Get_combined-16 139.0 ± 1% 137.0 ± 3% ~ (p=0.245 n=6) FSTree_Get/100KB/Get_compressed-16 140.0 ± 6% 139.0 ± 6% ~ (p=0.781 n=6) FSTree_Get/1MB/Get_regular-16 140.0 ± 6% 139.5 ± 5% ~ (p=0.900 n=6) FSTree_Get/1MB/Get_combined-16 136.5 ± 3% 139.0 ± 2% ~ (p=0.167 n=6) FSTree_Get/1MB/Get_compressed-16 143.0 ± 5% 133.0 ± 5% -6.99% (p=0.011 n=6) geomean 138.5 138.7 +0.11% │ oldHead.txt │ newHead.txt │ │ sec/op │ sec/op vs base │ FSTree_Head/Empty/Head_regular-16 85.25µ ± 3% 86.09µ ± 2% ~ (p=0.132 n=6) FSTree_Head/Empty/Head_combined-16 87.18µ ± 2% 88.98µ ± 1% +2.07% (p=0.009 n=6) FSTree_Head/Empty/Head_compressed-16 69.68µ ± 10% 86.31µ ± 2% +23.87% (p=0.002 n=6) FSTree_Head/100B/Head_regular-16 87.74µ ± 2% 85.86µ ± 4% ~ (p=0.394 n=6) FSTree_Head/100B/Head_combined-16 88.48µ ± 2% 89.27µ ± 2% +0.89% (p=0.041 n=6) FSTree_Head/100B/Head_compressed-16 72.00µ ± 5% 81.52µ ± 6% +13.24% (p=0.002 n=6) FSTree_Head/4KB/Head_regular-16 90.90µ ± 2% 90.57µ ± 5% ~ (p=0.937 n=6) FSTree_Head/4KB/Head_combined-16 98.41µ ± 3% 96.54µ ± 1% -1.90% (p=0.041 n=6) FSTree_Head/4KB/Head_compressed-16 76.73µ ± 5% 84.40µ ± 5% +9.99% (p=0.002 n=6) FSTree_Head/16KB/Head_regular-16 89.46µ ± 3% 91.33µ ± 3% +2.08% (p=0.041 n=6) FSTree_Head/16KB/Head_combined-16 117.8µ ± 2% 119.2µ ± 1% +1.19% (p=0.015 n=6) FSTree_Head/16KB/Head_compressed-16 154.23µ ± 3% 84.44µ ± 6% -45.25% (p=0.002 n=6) FSTree_Head/32KB/Head_regular-16 88.19µ ± 4% 97.20µ ± 3% +10.21% (p=0.002 n=6) FSTree_Head/32KB/Head_combined-16 118.0µ ± 3% 126.3µ ± 1% +6.98% (p=0.002 n=6) FSTree_Head/32KB/Head_compressed-16 175.94µ ± 3% 90.65µ ± 6% -48.48% (p=0.002 n=6) FSTree_Head/100KB/Head_regular-16 92.66µ ± 6% 99.57µ ± 3% +7.46% (p=0.002 n=6) FSTree_Head/100KB/Head_combined-16 118.8µ ± 1% 131.1µ ± 3% +10.36% (p=0.002 n=6) FSTree_Head/100KB/Head_compressed-16 241.29µ ± 1% 96.14µ ± 5% -60.16% (p=0.002 n=6) FSTree_Head/1MB/Head_regular-16 88.83µ ± 2% 93.01µ ± 3% +4.71% (p=0.009 n=6) FSTree_Head/1MB/Head_combined-16 109.5µ ± 21% 132.4µ ± 5% +20.96% (p=0.026 n=6) FSTree_Head/1MB/Head_compressed-16 799.82µ ± 5% 90.12µ ± 2% -88.73% (p=0.002 n=6) geomean 112.7µ 96.11µ -14.69% │ oldHead.txt │ newHead.txt │ │ B/op │ B/op vs base │ FSTree_Head/Empty/Head_regular-16 40.01Ki ± 1% 40.00Ki ± 1% ~ (p=0.974 n=6) FSTree_Head/Empty/Head_combined-16 40.06Ki ± 0% 40.07Ki ± 0% ~ (p=0.589 n=6) FSTree_Head/Empty/Head_compressed-16 41.64Ki ± 1% 40.14Ki ± 1% -3.60% (p=0.002 n=6) FSTree_Head/100B/Head_regular-16 40.39Ki ± 1% 39.82Ki ± 1% -1.41% (p=0.002 n=6) FSTree_Head/100B/Head_combined-16 40.21Ki ± 0% 39.84Ki ± 0% -0.93% (p=0.002 n=6) FSTree_Head/100B/Head_compressed-16 42.01Ki ± 1% 47.69Ki ± 1% +13.52% (p=0.002 n=6) FSTree_Head/4KB/Head_regular-16 44.10Ki ± 1% 39.96Ki ± 1% -9.38% (p=0.002 n=6) FSTree_Head/4KB/Head_combined-16 44.09Ki ± 0% 39.72Ki ± 1% -9.91% (p=0.002 n=6) FSTree_Head/4KB/Head_compressed-16 50.05Ki ± 1% 58.36Ki ± 1% +16.61% (p=0.002 n=6) FSTree_Head/16KB/Head_regular-16 39.69Ki ± 1% 39.77Ki ± 1% ~ (p=0.416 n=6) FSTree_Head/16KB/Head_combined-16 39.71Ki ± 0% 39.76Ki ± 0% ~ (p=0.240 n=6) FSTree_Head/16KB/Head_compressed-16 119.46Ki ± 0% 63.55Ki ± 1% -46.80% (p=0.002 n=6) FSTree_Head/32KB/Head_regular-16 39.41Ki ± 1% 39.83Ki ± 1% ~ (p=0.074 n=6) FSTree_Head/32KB/Head_combined-16 39.70Ki ± 0% 39.72Ki ± 0% ~ (p=0.699 n=6) FSTree_Head/32KB/Head_compressed-16 173.47Ki ± 0% 85.55Ki ± 1% -50.68% (p=0.002 n=6) FSTree_Head/100KB/Head_regular-16 40.00Ki ± 1% 39.73Ki ± 1% ~ (p=0.485 n=6) FSTree_Head/100KB/Head_combined-16 39.68Ki ± 0% 39.75Ki ± 0% ~ (p=0.093 n=6) FSTree_Head/100KB/Head_compressed-16 373.6Ki ± 0% 149.5Ki ± 0% -59.99% (p=0.002 n=6) FSTree_Head/1MB/Head_regular-16 39.67Ki ± 1% 39.74Ki ± 1% ~ (p=0.974 n=6) FSTree_Head/1MB/Head_combined-16 39.67Ki ± 0% 39.81Ki ± 0% +0.37% (p=0.004 n=6) FSTree_Head/1MB/Head_compressed-16 2661.4Ki ± 0% 181.5Ki ± 0% -93.18% (p=0.002 n=6) geomean 62.75Ki 49.68Ki -20.84% │ oldHead.txt │ newHead.txt │ │ allocs/op │ allocs/op vs base │ FSTree_Head/Empty/Head_regular-16 136.5 ± 7% 136.5 ± 7% ~ (p=0.751 n=6) FSTree_Head/Empty/Head_combined-16 138.0 ± 2% 138.0 ± 3% ~ (p=0.898 n=6) FSTree_Head/Empty/Head_compressed-16 141.5 ± 4% 141.0 ± 4% ~ (p=0.424 n=6) FSTree_Head/100B/Head_regular-16 145.0 ± 4% 141.5 ± 8% ~ (p=0.284 n=6) FSTree_Head/100B/Head_combined-16 140.0 ± 1% 142.5 ± 1% +1.79% (p=0.009 n=6) FSTree_Head/100B/Head_compressed-16 142.5 ± 6% 176.5 ± 5% +23.86% (p=0.002 n=6) FSTree_Head/4KB/Head_regular-16 141.5 ± 6% 146.0 ± 11% ~ (p=0.797 n=6) FSTree_Head/4KB/Head_combined-16 140.0 ± 3% 139.5 ± 3% ~ (p=0.900 n=6) FSTree_Head/4KB/Head_compressed-16 140.5 ± 5% 178.5 ± 9% +27.05% (p=0.002 n=6) FSTree_Head/16KB/Head_regular-16 139.5 ± 8% 141.5 ± 5% ~ (p=0.457 n=6) FSTree_Head/16KB/Head_combined-16 140.0 ± 4% 140.0 ± 3% ~ (p=0.745 n=6) FSTree_Head/16KB/Head_compressed-16 176.5 ± 6% 174.5 ± 4% ~ (p=0.660 n=6) FSTree_Head/32KB/Head_regular-16 132.0 ± 11% 143.0 ± 6% ~ (p=0.091 n=6) FSTree_Head/32KB/Head_combined-16 139.5 ± 3% 139.5 ± 3% ~ (p=0.905 n=6) FSTree_Head/32KB/Head_compressed-16 177.0 ± 7% 174.5 ± 6% ~ (p=0.810 n=6) FSTree_Head/100KB/Head_regular-16 148.5 ± 8% 139.5 ± 9% ~ (p=0.359 n=6) FSTree_Head/100KB/Head_combined-16 139.5 ± 3% 140.0 ± 1% ~ (p=0.697 n=6) FSTree_Head/100KB/Head_compressed-16 181.0 ± 7% 172.5 ± 6% ~ (p=0.158 n=6) FSTree_Head/1MB/Head_regular-16 139.5 ± 5% 139.5 ± 9% ~ (p=0.801 n=6) FSTree_Head/1MB/Head_combined-16 139.0 ± 2% 142.0 ± 2% +2.16% (p=0.024 n=6) FSTree_Head/1MB/Head_compressed-16 179.0 ± 4% 173.0 ± 3% -3.35% (p=0.028 n=6) geomean 146.7 149.7 +2.02% │ oldRange.txt │ newRange.txt │ │ sec/op │ sec/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 276.7µ ± 10% 258.5µ ± 5% ~ (p=0.180 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 273.1µ ± 5% 267.2µ ± 3% ~ (p=0.093 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 394.8µ ± 4% 330.9µ ± 8% -16.18% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 3.633m ± 4% 3.802m ± 3% +4.64% (p=0.004 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 3.661m ± 3% 3.770m ± 4% ~ (p=0.065 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 3.627m ± 5% 3.759m ± 4% ~ (p=0.132 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 3.660m ± 3% 3.831m ± 6% ~ (p=0.065 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 3.722m ± 9% 3.805m ± 3% ~ (p=0.240 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 3.625m ± 4% 3.722m ± 4% +2.69% (p=0.026 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 102.77µ ± 1% 97.48µ ± 1% -5.15% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 104.37µ ± 1% 97.82µ ± 1% -6.28% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 100.44µ ± 1% 97.27µ ± 4% -3.16% (p=0.041 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 980.1µ ± 4% 892.1µ ± 4% -8.97% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 972.1µ ± 4% 951.4µ ± 6% ~ (p=0.093 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 984.6µ ± 4% 1028.0µ ± 2% +4.41% (p=0.009 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 941.9µ ± 4% 842.8µ ± 6% -10.52% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 942.1µ ± 3% 859.7µ ± 10% -8.74% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 977.4µ ± 4% 1038.1µ ± 6% +6.22% (p=0.026 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 99.54µ ± 4% 100.73µ ± 3% +1.20% (p=0.041 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 99.09µ ± 2% 101.22µ ± 2% +2.15% (p=0.026 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 100.7µ ± 2% 100.4µ ± 2% ~ (p=0.485 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 102.78µ ± 2% 97.85µ ± 1% -4.79% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 102.10µ ± 1% 97.04µ ± 1% -4.96% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 100.2µ ± 3% 101.0µ ± 3% ~ (p=0.132 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 99.97µ ± 1% 103.14µ ± 1% +3.17% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 100.3µ ± 1% 103.1µ ± 3% ~ (p=0.132 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 100.47µ ± 3% 99.91µ ± 3% ~ (p=0.485 n=6) geomean 419.6µ 412.9µ -1.60% │ oldRange.txt │ newRange.txt │ │ B/op │ B/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 43.78Ki ± 0% 43.85Ki ± 0% +0.16% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 43.78Ki ± 0% 43.85Ki ± 0% +0.16% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 43.84Ki ± 0% 43.92Ki ± 0% +0.20% (p=0.041 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 10.04Mi ± 0% 10.04Mi ± 0% +0.00% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 10.04Mi ± 0% 10.04Mi ± 0% +0.00% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 10.04Mi ± 0% 10.04Mi ± 0% +0.00% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 10.04Mi ± 0% 10.04Mi ± 0% +0.01% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 10.04Mi ± 0% 10.04Mi ± 0% +0.01% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 10.04Mi ± 0% 10.04Mi ± 0% +0.00% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 44.23Ki ± 0% 44.01Ki ± 0% -0.51% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 44.24Ki ± 0% 44.01Ki ± 0% -0.52% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 43.92Ki ± 0% 43.99Ki ± 0% +0.15% (p=0.041 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 1.039Mi ± 0% 1.039Mi ± 0% +0.01% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 1.039Mi ± 0% 1.039Mi ± 0% +0.00% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 1.039Mi ± 0% 1.039Mi ± 0% ~ (p=0.065 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 1.039Mi ± 0% 1.039Mi ± 0% +0.02% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 1.039Mi ± 0% 1.039Mi ± 0% +0.02% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 1.039Mi ± 0% 1.039Mi ± 0% ~ (p=0.394 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 48.00Ki ± 0% 43.98Ki ± 0% -8.38% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 48.00Ki ± 0% 43.98Ki ± 0% -8.38% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 48.11Ki ± 0% 43.85Ki ± 0% -8.84% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 48.41Ki ± 0% 43.63Ki ± 0% -9.86% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 48.41Ki ± 0% 43.63Ki ± 0% -9.86% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 48.09Ki ± 0% 43.87Ki ± 0% -8.78% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 45.09Ki ± 0% 41.41Ki ± 0% -8.16% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 45.09Ki ± 0% 41.40Ki ± 0% -8.17% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 45.16Ki ± 0% 40.96Ki ± 0% -9.29% (p=0.002 n=6) geomean 307.0Ki 297.6Ki -3.06% │ oldRange.txt │ newRange.txt │ │ allocs/op │ allocs/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 142.0 ± 0% 143.0 ± 0% +0.70% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 142.0 ± 0% 143.0 ± 0% +0.70% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 141.5 ± 3% 143.0 ± 1% ~ (p=0.141 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 137.0 ± 0% 147.0 ± 0% +7.30% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 137.0 ± 0% 147.0 ± 0% +7.30% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 140.5 ± 0% 143.5 ± 2% +2.14% (p=0.022 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 129.0 ± 0% 146.0 ± 0% +13.18% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 129.0 ± 0% 146.0 ± 0% +13.18% (p=0.002 n=6) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 138.5 ± 2% 143.0 ± 1% +3.25% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 149.0 ± 0% 143.0 ± 0% -4.03% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 149.0 ± 0% 143.0 ± 0% -4.03% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 141.5 ± 1% 142.5 ± 4% ~ (p=0.262 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 144.0 ± 0% 146.0 ± 0% +1.39% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 144.0 ± 0% 146.0 ± 0% +1.39% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 141.5 ± 0% 142.0 ± 3% ~ (p=0.457 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 145.0 ± 0% 150.0 ± 0% +3.45% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 145.0 ± 0% 150.0 ± 0% +3.45% (p=0.002 n=6) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 141.5 ± 1% 141.5 ± 1% ~ (p=0.994 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 137.0 ± 0% 146.0 ± 0% +6.57% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 137.0 ± 0% 146.0 ± 0% +6.57% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 139.5 ± 2% 142.0 ± 2% ~ (p=0.082 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 148.0 ± 0% 135.0 ± 0% -8.78% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 148.0 ± 0% 135.0 ± 0% -8.78% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 139.0 ± 3% 142.5 ± 1% +2.52% (p=0.037 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 140.0 ± 0% 154.0 ± 0% +10.00% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 140.0 ± 0% 154.0 ± 0% +10.00% (p=0.002 n=6) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 141.0 ± 3% 142.5 ± 2% ~ (p=0.355 n=6) geomean 140.9 144.5 +2.54% │ oldStream.txt │ newStream.txt │ │ sec/op │ sec/op vs base │ FSTree_GetStream/Empty/GetStream_regular-16 76.45µ ± 4% 73.36µ ± 6% ~ (p=0.093 n=6) FSTree_GetStream/Empty/GetStream_combined-16 78.77µ ± 6% 75.85µ ± 6% ~ (p=0.132 n=6) FSTree_GetStream/Empty/GetStream_compressed-16 60.46µ ± 12% 74.30µ ± 5% +22.90% (p=0.002 n=6) FSTree_GetStream/Empty/GetStream_with_payload_read-16 58.42µ ± 10% 78.13µ ± 4% +33.74% (p=0.002 n=6) FSTree_GetStream/100B/GetStream_regular-16 77.34µ ± 5% 78.40µ ± 6% ~ (p=0.485 n=6) FSTree_GetStream/100B/GetStream_combined-16 79.85µ ± 3% 79.55µ ± 5% ~ (p=0.937 n=6) FSTree_GetStream/100B/GetStream_compressed-16 58.35µ ± 12% 63.90µ ± 8% ~ (p=0.065 n=6) FSTree_GetStream/100B/GetStream_with_payload_read-16 58.50µ ± 10% 96.01µ ± 7% +64.10% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_regular-16 80.55µ ± 5% 87.08µ ± 6% +8.09% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_combined-16 88.60µ ± 2% 93.28µ ± 8% +5.28% (p=0.004 n=6) FSTree_GetStream/4KB/GetStream_compressed-16 62.76µ ± 13% 67.13µ ± 6% ~ (p=0.180 n=6) FSTree_GetStream/4KB/GetStream_with_payload_read-16 69.95µ ± 6% 126.12µ ± 9% +80.30% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_regular-16 78.75µ ± 6% 97.42µ ± 5% +23.70% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_combined-16 102.8µ ± 3% 134.3µ ± 3% +30.70% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_compressed-16 142.49µ ± 7% 67.82µ ± 31% -52.40% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_with_payload_read-16 166.8µ ± 5% 158.1µ ± 5% -5.22% (p=0.015 n=6) FSTree_GetStream/32KB/GetStream_regular-16 81.14µ ± 5% 82.27µ ± 6% ~ (p=0.818 n=6) FSTree_GetStream/32KB/GetStream_combined-16 105.0µ ± 5% 108.3µ ± 4% ~ (p=0.240 n=6) FSTree_GetStream/32KB/GetStream_compressed-16 164.59µ ± 6% 73.07µ ± 6% -55.61% (p=0.002 n=6) FSTree_GetStream/32KB/GetStream_with_payload_read-16 202.3µ ± 5% 193.8µ ± 3% -4.19% (p=0.002 n=6) FSTree_GetStream/100KB/GetStream_regular-16 81.49µ ± 6% 83.25µ ± 2% ~ (p=0.558 n=6) FSTree_GetStream/100KB/GetStream_combined-16 111.0µ ± 6% 108.3µ ± 3% ~ (p=0.065 n=6) FSTree_GetStream/100KB/GetStream_compressed-16 212.90µ ± 6% 81.45µ ± 4% -61.74% (p=0.002 n=6) FSTree_GetStream/100KB/GetStream_with_payload_read-16 356.5µ ± 4% 341.1µ ± 8% ~ (p=0.132 n=6) FSTree_GetStream/1MB/GetStream_regular-16 79.85µ ± 9% 85.54µ ± 5% +7.12% (p=0.026 n=6) FSTree_GetStream/1MB/GetStream_combined-16 112.4µ ± 5% 111.2µ ± 4% ~ (p=0.240 n=6) FSTree_GetStream/1MB/GetStream_compressed-16 778.17µ ± 6% 78.41µ ± 6% -89.92% (p=0.002 n=6) FSTree_GetStream/1MB/GetStream_with_payload_read-16 2.015m ± 8% 2.161m ± 3% +7.25% (p=0.009 n=6) geomean 116.6µ 106.7µ -8.54% │ oldStream.txt │ newStream.txt │ │ B/op │ B/op vs base │ FSTree_GetStream/Empty/GetStream_regular-16 40.13Ki ± 1% 39.95Ki ± 1% ~ (p=0.290 n=6) FSTree_GetStream/Empty/GetStream_combined-16 40.11Ki ± 0% 40.07Ki ± 0% ~ (p=0.143 n=6) FSTree_GetStream/Empty/GetStream_compressed-16 41.56Ki ± 1% 40.17Ki ± 1% -3.35% (p=0.002 n=6) FSTree_GetStream/Empty/GetStream_with_payload_read-16 41.91Ki ± 1% 40.68Ki ± 1% -2.93% (p=0.002 n=6) FSTree_GetStream/100B/GetStream_regular-16 40.16Ki ± 1% 39.88Ki ± 1% ~ (p=0.180 n=6) FSTree_GetStream/100B/GetStream_combined-16 40.11Ki ± 0% 39.76Ki ± 0% -0.89% (p=0.002 n=6) FSTree_GetStream/100B/GetStream_compressed-16 42.00Ki ± 1% 47.62Ki ± 1% +13.37% (p=0.002 n=6) FSTree_GetStream/100B/GetStream_with_payload_read-16 42.39Ki ± 1% 48.31Ki ± 1% +13.98% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_regular-16 43.91Ki ± 0% 39.71Ki ± 1% -9.55% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_combined-16 44.09Ki ± 0% 39.78Ki ± 0% -9.78% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_compressed-16 50.27Ki ± 1% 58.05Ki ± 0% +15.49% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_with_payload_read-16 67.14Ki ± 1% 75.38Ki ± 1% +12.26% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_regular-16 39.61Ki ± 1% 39.90Ki ± 1% ~ (p=0.394 n=6) FSTree_GetStream/16KB/GetStream_combined-16 39.65Ki ± 0% 39.78Ki ± 0% +0.32% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_compressed-16 119.68Ki ± 0% 63.56Ki ± 1% -46.89% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_with_payload_read-16 201.8Ki ± 0% 177.8Ki ± 0% -11.87% (p=0.002 n=6) FSTree_GetStream/32KB/GetStream_regular-16 39.52Ki ± 1% 39.59Ki ± 1% ~ (p=0.589 n=6) FSTree_GetStream/32KB/GetStream_combined-16 39.72Ki ± 1% 39.77Ki ± 0% ~ (p=0.093 n=6) FSTree_GetStream/32KB/GetStream_compressed-16 173.43Ki ± 0% 85.50Ki ± 0% -50.70% (p=0.002 n=6) FSTree_GetStream/32KB/GetStream_with_payload_read-16 323.8Ki ± 0% 299.7Ki ± 0% -7.44% (p=0.002 n=6) FSTree_GetStream/100KB/GetStream_regular-16 39.66Ki ± 1% 39.89Ki ± 1% ~ (p=0.143 n=6) FSTree_GetStream/100KB/GetStream_combined-16 39.67Ki ± 0% 39.82Ki ± 0% +0.36% (p=0.002 n=6) FSTree_GetStream/100KB/GetStream_compressed-16 373.2Ki ± 0% 149.5Ki ± 0% -59.93% (p=0.002 n=6) FSTree_GetStream/100KB/GetStream_with_payload_read-16 875.7Ki ± 0% 851.7Ki ± 0% -2.74% (p=0.002 n=6) FSTree_GetStream/1MB/GetStream_regular-16 39.84Ki ± 1% 39.84Ki ± 1% ~ (p=0.740 n=6) FSTree_GetStream/1MB/GetStream_combined-16 39.67Ki ± 0% 39.77Ki ± 0% +0.25% (p=0.022 n=6) FSTree_GetStream/1MB/GetStream_compressed-16 2661.4Ki ± 0% 181.7Ki ± 0% -93.17% (p=0.002 n=6) FSTree_GetStream/1MB/GetStream_with_payload_read-16 7.597Mi ± 0% 7.574Mi ± 0% -0.30% (p=0.002 n=6) geomean 88.18Ki 73.97Ki -16.11% │ oldStream.txt │ newStream.txt │ │ allocs/op │ allocs/op vs base │ FSTree_GetStream/Empty/GetStream_regular-16 140.5 ± 7% 135.0 ± 6% ~ (p=0.201 n=6) FSTree_GetStream/Empty/GetStream_combined-16 139.0 ± 1% 138.0 ± 2% ~ (p=0.253 n=6) FSTree_GetStream/Empty/GetStream_compressed-16 139.5 ± 8% 142.5 ± 8% ~ (p=0.959 n=6) FSTree_GetStream/Empty/GetStream_with_payload_read-16 138.5 ± 7% 143.0 ± 5% ~ (p=0.221 n=6) FSTree_GetStream/100B/GetStream_regular-16 140.0 ± 6% 143.5 ± 7% ~ (p=0.381 n=6) FSTree_GetStream/100B/GetStream_combined-16 138.0 ± 3% 140.0 ± 4% ~ (p=0.076 n=6) FSTree_GetStream/100B/GetStream_compressed-16 142.5 ± 7% 175.0 ± 7% +22.81% (p=0.002 n=6) FSTree_GetStream/100B/GetStream_with_payload_read-16 140.0 ± 6% 181.0 ± 7% +29.29% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_regular-16 136.0 ± 3% 140.0 ± 4% ~ (p=0.275 n=6) FSTree_GetStream/4KB/GetStream_combined-16 139.5 ± 3% 141.0 ± 1% ~ (p=0.206 n=6) FSTree_GetStream/4KB/GetStream_compressed-16 146.0 ± 9% 169.0 ± 5% +15.75% (p=0.002 n=6) FSTree_GetStream/4KB/GetStream_with_payload_read-16 150.0 ± 7% 185.5 ± 8% +23.67% (p=0.002 n=6) FSTree_GetStream/16KB/GetStream_regular-16 138.0 ± 7% 145.5 ± 8% ~ (p=0.370 n=6) FSTree_GetStream/16KB/GetStream_combined-16 139.0 ± 2% 141.0 ± 2% +1.44% (p=0.026 n=6) FSTree_GetStream/16KB/GetStream_compressed-16 182.0 ± 3% 174.5 ± 4% ~ (p=0.065 n=6) FSTree_GetStream/16KB/GetStream_with_payload_read-16 189.0 ± 4% 188.0 ± 3% ~ (p=0.820 n=6) FSTree_GetStream/32KB/GetStream_regular-16 136.0 ± 6% 136.0 ± 7% ~ (p=0.775 n=6) FSTree_GetStream/32KB/GetStream_combined-16 140.5 ± 5% 140.5 ± 2% ~ (p=0.502 n=6) FSTree_GetStream/32KB/GetStream_compressed-16 175.0 ± 7% 174.5 ± 4% ~ (p=0.779 n=6) FSTree_GetStream/32KB/GetStream_with_payload_read-16 192.5 ± 3% 186.5 ± 5% ~ (p=0.167 n=6) FSTree_GetStream/100KB/GetStream_regular-16 139.5 ± 8% 144.0 ± 4% ~ (p=0.290 n=6) FSTree_GetStream/100KB/GetStream_combined-16 139.5 ± 2% 141.5 ± 1% +1.43% (p=0.022 n=6) FSTree_GetStream/100KB/GetStream_compressed-16 171.0 ± 5% 175.0 ± 5% ~ (p=0.348 n=6) FSTree_GetStream/100KB/GetStream_with_payload_read-16 194.0 ± 6% 190.5 ± 7% ~ (p=0.775 n=6) FSTree_GetStream/1MB/GetStream_regular-16 143.5 ± 5% 143.5 ± 6% ~ (p=1.000 n=6) FSTree_GetStream/1MB/GetStream_combined-16 139.0 ± 2% 141.0 ± 1% ~ (p=0.245 n=6) FSTree_GetStream/1MB/GetStream_compressed-16 177.5 ± 5% 171.0 ± 6% ~ (p=0.084 n=6) FSTree_GetStream/1MB/GetStream_with_payload_read-16 206.5 ± 4% 212.0 ± 3% +2.66% (p=0.019 n=6) geomean 151.9 157.1 +3.43% │ oldPut.txt │ newPut.txt │ │ sec/op │ sec/op vs base │ Put/size=1,thread=1/fstree-16 15.85m ± 7% 15.68m ± 2% ~ (p=0.180 n=6) Put/size=1,thread=20/fstree-16 18.90m ± 25% 17.06m ± 7% ~ (p=0.065 n=6) Put/size=1,thread=100/fstree-16 23.64m ± 5% 22.74m ± 32% ~ (p=0.589 n=6) Put/size=1024,thread=1/fstree-16 16.11m ± 5% 16.05m ± 4% ~ (p=0.485 n=6) Put/size=1024,thread=20/fstree-16 17.77m ± 25% 17.88m ± 3% ~ (p=1.000 n=6) Put/size=1024,thread=100/fstree-16 24.27m ± 10% 23.49m ± 15% ~ (p=0.485 n=6) Put/size=102400,thread=1/fstree-16 16.59m ± 8% 16.49m ± 3% ~ (p=0.394 n=6) Put/size=102400,thread=20/fstree-16 31.49m ± 25% 27.05m ± 9% ~ (p=0.093 n=6) Put/size=102400,thread=100/fstree-16 175.4m ± 39% 178.4m ± 18% ~ (p=0.485 n=6) geomean 25.47m 24.57m -3.52% │ oldPut.txt │ newPut.txt │ │ B/op │ B/op vs base │ Put/size=1,thread=1/fstree-16 3.405Ki ± 1% 3.437Ki ± 1% +0.93% (p=0.013 n=6) Put/size=1,thread=20/fstree-16 55.81Ki ± 4% 57.10Ki ± 4% ~ (p=0.132 n=6) Put/size=1,thread=100/fstree-16 260.5Ki ± 2% 268.7Ki ± 2% +3.16% (p=0.009 n=6) Put/size=1024,thread=1/fstree-16 3.388Ki ± 1% 3.425Ki ± 0% +1.10% (p=0.002 n=6) Put/size=1024,thread=20/fstree-16 55.82Ki ± 1% 56.81Ki ± 3% +1.78% (p=0.002 n=6) Put/size=1024,thread=100/fstree-16 264.2Ki ± 1% 268.4Ki ± 2% +1.60% (p=0.004 n=6) Put/size=102400,thread=1/fstree-16 3.419Ki ± 0% 3.464Ki ± 1% +1.30% (p=0.002 n=6) Put/size=102400,thread=20/fstree-16 57.58Ki ± 2% 58.39Ki ± 1% +1.40% (p=0.002 n=6) Put/size=102400,thread=100/fstree-16 290.5Ki ± 2% 296.2Ki ± 2% +1.96% (p=0.026 n=6) geomean 37.35Ki 37.99Ki +1.72% │ oldPut.txt │ newPut.txt │ │ allocs/op │ allocs/op vs base │ Put/size=1,thread=1/fstree-16 37.00 ± 3% 38.50 ± 1% +4.05% (p=0.002 n=6) Put/size=1,thread=20/fstree-16 559.0 ± 1% 597.0 ± 2% +6.80% (p=0.002 n=6) Put/size=1,thread=100/fstree-16 2.596k ± 1% 2.800k ± 4% +7.84% (p=0.002 n=6) Put/size=1024,thread=1/fstree-16 37.00 ± 3% 38.00 ± 3% +2.70% (p=0.002 n=6) Put/size=1024,thread=20/fstree-16 556.0 ± 1% 595.5 ± 4% +7.10% (p=0.002 n=6) Put/size=1024,thread=100/fstree-16 2.603k ± 1% 2.803k ± 0% +7.64% (p=0.002 n=6) Put/size=102400,thread=1/fstree-16 37.00 ± 3% 39.00 ± 3% +5.41% (p=0.002 n=6) Put/size=102400,thread=20/fstree-16 569.0 ± 1% 607.5 ± 1% +6.77% (p=0.002 n=6) Put/size=102400,thread=100/fstree-16 2.854k ± 2% 3.070k ± 3% +7.55% (p=0.002 n=6) geomean 381.9 405.6 +6.19% ``` Signed-off-by: Andrey Butusov <[email protected]>
1 parent 77e2942 commit 4a3982e

File tree

3 files changed

+191
-41
lines changed

3 files changed

+191
-41
lines changed

‎pkg/local_object_storage/blobstor/fstree/fstree.go‎

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
2323
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
2424
"go.uber.org/zap"
25+
"google.golang.org/protobuf/encoding/protowire"
2526
)
2627

2728
// FSTree represents an object storage as a filesystem tree.
@@ -90,6 +91,26 @@ const (
9091
// combinedDataOff is the offset from the start of the combined prefix to object data.
9192
// It's also the length of the prefix in total.
9293
combinedDataOff = combinedLengthOff + combinedLenSize
94+
95+
// streamPrefix is the prefix for streamed objects. It is used to distinguish
96+
// streamed objects from regular ones.
97+
streamPrefix = 0x7e
98+
99+
// streamLenHeaderOff is the offset from the start of the stream prefix to
100+
// the length of the header data.
101+
streamLenHeaderOff = 2
102+
103+
// streamLenSize is sizeof(uint32), length of a serialized 32-bit BE integer
104+
// that represents the length of the header or payload data.
105+
streamLenSize = 4
106+
107+
// streamLenDataOff is the offset from the start of the stream prefix to
108+
// the length of the data.
109+
streamLenDataOff = streamLenHeaderOff + streamLenSize
110+
111+
// streamDataOff is the offset from the start of the stream prefix to the
112+
// start of the data. It is used to read the data after the header.
113+
streamDataOff = streamLenDataOff + streamLenSize
93114
)
94115

95116
var _ common.Storage = (*FSTree)(nil)
@@ -339,7 +360,7 @@ func (t *FSTree) Put(addr oid.Address, data []byte) error {
339360
if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil {
340361
return fmt.Errorf("mkdirall for %q: %w", p, err)
341362
}
342-
data = t.Compress(data)
363+
data = t.processHeaderAndPayload(data)
343364

344365
err := t.writer.writeData(addr.Object(), p, data)
345366
if err != nil {
@@ -363,7 +384,7 @@ func (t *FSTree) PutBatch(objs map[oid.Address][]byte) error {
363384
writeDataUnits = append(writeDataUnits, writeDataUnit{
364385
id: addr.Object(),
365386
path: p,
366-
data: t.Compress(data),
387+
data: t.processHeaderAndPayload(data),
367388
})
368389
}
369390

@@ -375,6 +396,32 @@ func (t *FSTree) PutBatch(objs map[oid.Address][]byte) error {
375396
return nil
376397
}
377398

399+
// processHeaderAndPayload processes the header and payload of the object data.
400+
func (t *FSTree) processHeaderAndPayload(data []byte) []byte {
401+
headerEnd, payloadStart, err := extractHeaderAndPayload(data, nil)
402+
if err != nil || headerEnd == 0 {
403+
return data
404+
}
405+
406+
header := data[:headerEnd]
407+
payload := data[payloadStart:]
408+
409+
hLen := len(header)
410+
payload = t.Compress(payload)
411+
pLen := len(payload)
412+
413+
res := make([]byte, hLen+pLen+streamDataOff)
414+
res[0] = streamPrefix
415+
res[1] = 0 // version 0
416+
binary.BigEndian.PutUint32(res[streamLenHeaderOff:], uint32(hLen))
417+
binary.BigEndian.PutUint32(res[streamLenDataOff:], uint32(pLen))
418+
419+
copy(res[streamDataOff:], header)
420+
copy(res[streamDataOff+hLen:], payload)
421+
422+
return res
423+
}
424+
378425
// Get returns an object from the storage by address.
379426
func (t *FSTree) Get(addr oid.Address) (*objectSDK.Object, error) {
380427
data, err := t.getObjBytes(addr)
@@ -433,6 +480,16 @@ func parseCombinedPrefix(p []byte) ([]byte, uint32) {
433480
binary.BigEndian.Uint32(p[combinedLengthOff:combinedDataOff])
434481
}
435482

483+
// parseStreamPrefix checks the given byte slice for stream prefix and returns
484+
// the length of the header and data if so (0, 0 otherwise).
485+
func parseStreamPrefix(p []byte) (uint32, uint32) {
486+
if p[0] != streamPrefix || p[1] != 0 { // Only version 0 is supported now.
487+
return 0, 0
488+
}
489+
return binary.BigEndian.Uint32(p[streamLenHeaderOff:streamLenDataOff]),
490+
binary.BigEndian.Uint32(p[streamLenDataOff:])
491+
}
492+
436493
func (t *FSTree) extractCombinedObject(id oid.ID, f *os.File) ([]byte, error) {
437494
var (
438495
comBuf [combinedDataOff]byte
@@ -485,6 +542,23 @@ func (t *FSTree) readFullObject(f io.Reader, initial []byte, size int64) ([]byte
485542
return nil, fmt.Errorf("read: %w", err)
486543
}
487544
data = data[:len(initial)+n]
545+
hLen, _ := parseStreamPrefix(data)
546+
if hLen > 0 {
547+
data = data[streamDataOff:]
548+
payload, err := t.Decompress(data[hLen:])
549+
if err != nil {
550+
return nil, fmt.Errorf("decompress payload: %w", err)
551+
}
552+
pLen := len(payload)
553+
payloadNum := protowire.Number(4)
554+
n := protowire.SizeTag(payloadNum) + protowire.SizeVarint(uint64(pLen))
555+
buf := make([]byte, int(hLen)+pLen+n)
556+
copy(buf[:hLen], data)
557+
off := binary.PutUvarint(buf[hLen:], protowire.EncodeTag(payloadNum, protowire.BytesType)) + int(hLen)
558+
off += binary.PutUvarint(buf[off:], uint64(pLen))
559+
copy(buf[off:], payload)
560+
data = buf
561+
}
488562

489563
return t.Decompress(data)
490564
}

‎pkg/local_object_storage/blobstor/fstree/getstream_test.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func TestGetStreamAfterErrors(t *testing.T) {
100100
t.Run("corrupt compressed data", func(t *testing.T) {
101101
compress := compression.Config{Enabled: true}
102102
require.NoError(t, compress.Init())
103-
tree.Config = &compress
103+
tree.SetCompressor(&compress)
104104

105105
addr := oidtest.Address()
106106
obj := objectSDK.New()
@@ -118,7 +118,16 @@ func TestGetStreamAfterErrors(t *testing.T) {
118118
require.NoError(t, err)
119119
require.NoError(t, f.Close())
120120

121-
_, _, err = tree.GetStream(addr)
122-
require.Error(t, err)
121+
res, reader, err := tree.GetStream(addr)
122+
require.NoError(t, err)
123+
require.NotNil(t, res)
124+
require.Equal(t, res.CutPayload(), res)
125+
require.NotNil(t, reader)
126+
127+
streamedPayload, err := io.ReadAll(reader)
128+
// we use io.LimitReader to avoid reading the corrupted part
129+
require.NoError(t, err)
130+
require.Equal(t, streamedPayload, payload)
131+
require.NoError(t, reader.Close())
123132
})
124133
}

‎pkg/local_object_storage/blobstor/fstree/head.go‎

Lines changed: 103 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,57 @@ func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Objec
124124

125125
// readHeaderAndPayload reads an object header from the file and returns reader for payload.
126126
// This function takes ownership of the io.ReadCloser and will close it if it does not return it.
127-
func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
127+
func (t *FSTree) readHeaderAndPayload(f io.ReadSeekCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
128128
var err error
129+
var hLen, pLen uint32
130+
if len(initial) >= streamDataOff {
131+
hLen, pLen = parseStreamPrefix(initial)
132+
} else {
133+
var p []byte
134+
copy(p[:], initial)
135+
_, err := io.ReadFull(f, p[len(initial):])
136+
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
137+
return nil, f, fmt.Errorf("read stream prefix: %w", err)
138+
}
139+
hLen, pLen = parseStreamPrefix(p)
140+
if hLen == 0 {
141+
initial = p[:]
142+
}
143+
}
144+
if hLen > 0 {
145+
initial = initial[streamDataOff:]
146+
var header []byte
147+
if len(initial) < int(hLen) {
148+
header = make([]byte, hLen)
149+
copy(header, initial)
150+
_, err = io.ReadFull(f, header[len(initial):])
151+
if err != nil {
152+
return nil, nil, fmt.Errorf("read stream header: %w", err)
153+
}
154+
initial = header
155+
}
156+
header = initial[:hLen]
157+
var obj objectSDK.Object
158+
err = obj.Unmarshal(header)
159+
if err != nil {
160+
return nil, nil, fmt.Errorf("unmarshal object: %w", err)
161+
}
162+
163+
data := initial[hLen:]
164+
reader := io.LimitReader(io.MultiReader(bytes.NewReader(data), f), int64(pLen))
165+
if t.IsCompressed(data) {
166+
decoder, err := zstd.NewReader(reader)
167+
if err != nil {
168+
return nil, nil, fmt.Errorf("zstd decoder: %w", err)
169+
}
170+
reader = decoder.IOReadCloser()
171+
}
172+
return &obj, &payloadReader{
173+
Reader: reader,
174+
close: f.Close,
175+
}, nil
176+
}
177+
129178
if len(initial) < objectSDK.MaxHeaderLen {
130179
_ = f.Close()
131180
initial, err = t.Decompress(initial)
@@ -168,80 +217,98 @@ func (t *FSTree) readUntilPayload(f io.ReadCloser, initial []byte) (*objectSDK.O
168217
initial = buf[:n]
169218
}
170219

171-
obj, rest, err := extractHeaderAndPayload(initial)
220+
var (
221+
obj object.Object
222+
res objectSDK.Object
223+
)
224+
225+
_, offset, err := extractHeaderAndPayload(initial, func(num int, val []byte) error {
226+
switch num {
227+
case fieldObjectID:
228+
obj.ObjectId = new(refs.ObjectID)
229+
err := proto.Unmarshal(val, obj.ObjectId)
230+
if err != nil {
231+
return fmt.Errorf("unmarshal object ID: %w", err)
232+
}
233+
case fieldObjectSignature:
234+
obj.Signature = new(refs.Signature)
235+
err := proto.Unmarshal(val, obj.Signature)
236+
if err != nil {
237+
return fmt.Errorf("unmarshal object signature: %w", err)
238+
}
239+
case fieldObjectHeader:
240+
obj.Header = new(object.Header)
241+
err := proto.Unmarshal(val, obj.Header)
242+
if err != nil {
243+
return fmt.Errorf("unmarshal object header: %w", err)
244+
}
245+
default:
246+
return fmt.Errorf("unknown field number: %d", num)
247+
}
248+
return nil
249+
})
172250
if err != nil {
173251
_ = reader.Close()
174252
return nil, nil, fmt.Errorf("extract header and payload: %w", err)
175253
}
176254

177-
return obj, &payloadReader{
178-
Reader: io.MultiReader(bytes.NewReader(rest), reader),
255+
err = res.FromProtoMessage(&obj)
256+
if err != nil {
257+
_ = reader.Close()
258+
return nil, nil, fmt.Errorf("convert to objectSDK.Object: %w", err)
259+
}
260+
261+
return &res, &payloadReader{
262+
Reader: io.MultiReader(bytes.NewReader(initial[offset:]), reader),
179263
close: reader.Close,
180264
}, nil
181265
}
182266

183-
// extractHeaderAndPayload extracts the header of an object from the given byte slice and returns rest of the data.
184-
func extractHeaderAndPayload(data []byte) (*objectSDK.Object, []byte, error) {
185-
var (
186-
offset int
187-
res objectSDK.Object
188-
obj object.Object
189-
)
267+
// extractHeaderAndPayload processes the initial data to extract the header and payload
268+
// fields of an object. It calls the provided dataHandler for each field found in the data.
269+
// It returns the start offset of the header, the end offset of the payload, and an error if any.
270+
func extractHeaderAndPayload(data []byte, dataHandler func(int, []byte) error) (int, int, error) {
271+
var offset, headerEnd int
190272

191273
if len(data) == 0 {
192-
return nil, nil, fmt.Errorf("empty data")
274+
return 0, 0, fmt.Errorf("empty data")
193275
}
194276

195277
for offset < len(data) {
196278
num, typ, n := protowire.ConsumeTag(data[offset:])
197279
if err := protowire.ParseError(n); err != nil {
198-
return nil, nil, fmt.Errorf("invalid tag at offset %d: %w", offset, err)
280+
return 0, 0, fmt.Errorf("invalid tag at offset %d: %w", offset, err)
199281
}
200282
offset += n
201283

202284
if typ != protowire.BytesType {
203-
return nil, nil, fmt.Errorf("unexpected wire type: %v", typ)
285+
return 0, 0, fmt.Errorf("unexpected wire type: %v", typ)
204286
}
205287

206288
if num == fieldObjectPayload {
289+
headerEnd = offset - n
207290
_, n = binary.Varint(data[offset:])
208291
if err := protowire.ParseError(n); err != nil {
209-
return nil, nil, fmt.Errorf("invalid varint at offset %d: %w", offset, err)
292+
return 0, 0, fmt.Errorf("invalid varint at offset %d: %w", offset, err)
210293
}
211294
offset += n
212295
break
213296
}
214297
val, n := protowire.ConsumeBytes(data[offset:])
215298
if err := protowire.ParseError(n); err != nil {
216-
return nil, nil, fmt.Errorf("invalid bytes field at offset %d: %w", offset, err)
299+
return 0, 0, fmt.Errorf("invalid bytes field at offset %d: %w", offset, err)
217300
}
218301
offset += n
219302

220-
switch num {
221-
case fieldObjectID:
222-
obj.ObjectId = new(refs.ObjectID)
223-
err := proto.Unmarshal(val, obj.ObjectId)
224-
if err != nil {
225-
return nil, nil, fmt.Errorf("unmarshal object ID: %w", err)
226-
}
227-
case fieldObjectSignature:
228-
obj.Signature = new(refs.Signature)
229-
err := proto.Unmarshal(val, obj.Signature)
303+
if dataHandler != nil {
304+
err := dataHandler(int(num), val)
230305
if err != nil {
231-
return nil, nil, fmt.Errorf("unmarshal object signature: %w", err)
306+
return 0, 0, fmt.Errorf("data handler error at offset %d: %w", offset, err)
232307
}
233-
case fieldObjectHeader:
234-
obj.Header = new(object.Header)
235-
err := proto.Unmarshal(val, obj.Header)
236-
if err != nil {
237-
return nil, nil, fmt.Errorf("unmarshal object header: %w", err)
238-
}
239-
default:
240-
return nil, nil, fmt.Errorf("unknown field number: %d", num)
241308
}
242309
}
243310

244-
return &res, data[offset:], res.FromProtoMessage(&obj)
311+
return headerEnd, offset, nil
245312
}
246313

247314
type payloadReader struct {

0 commit comments

Comments
 (0)