From 49b2f2b5c02cd64fbbdb0abd0d15edfbee79023a Mon Sep 17 00:00:00 2001 From: Mathias Bogaert Date: Fri, 21 Nov 2025 22:55:22 +0000 Subject: [PATCH] Add bzip2 archive support for unpack parameter Extends archive extraction to support bzip2 compressed files (.bz2) and bzip2 compressed tarballs (.tar.bz2, .tbz2, .tb2) in addition to the existing gzip support. Implementation: - Added application/x-bzip2 and application/x-bzip MIME type detection - Added bunzip2 decompression handler in inflate() - Extended extractArchive() to handle two-stage bzip2 extraction - Mirrors existing gzip pattern for consistency Supported formats after this change: - .tar, .tar.gz/.tgz, .tar.bz2/.tbz2/.tb2, .gz, .bz2, .zip Updated documentation in README.md to reflect new capabilities. Fixes #81 Signed-off-by: Mathias Bogaert --- Dockerfile | 1 + README.md | 4 +- go.mod | 69 +++++++------ go.sum | 74 ++++++++++++++ in/archive.go | 5 + in/command.go | 12 ++- in/command_test.go | 96 ++++++++++++++++++ integration/in_test.go | 222 +++++++++++++++++++++++++++++++++++++++++ s3client_test.go | 1 + 9 files changed, 448 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index 25fbad36..b5f8e75c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ FROM ${base_image} AS resource RUN apk --no-cache add \ tzdata \ ca-certificates \ + cmd:bunzip2 \ cmd:unzip \ cmd:tar \ cmd:gunzip diff --git a/README.md b/README.md index d67c4bb1..c32f659f 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Places the following files in the destination: * `skip_download`: *Optional.* Skip downloading object from S3. Same parameter as source configuration but used to define/override by get. Value needs to be a true/false string. -* `unpack`: *Optional.* If true and the file is an archive (tar, gzipped tar, other gzipped file, or zip), unpack the file. Gzipped tarballs will be both ungzipped and untarred. It is ignored when `get` is running on the initial version. +* `unpack`: *Optional.* If true and the file is an archive (tar, gzipped tar, bzip2 compressed tar, other gzipped file, other bzip2 compressed file, or zip), unpack the file. Gzipped and bzip2 compressed tarballs will be both decompressed and untarred. It is ignored when `get` is running on the initial version. * `download_tags`: *Optional.* Write object tags to `tags.json`. Value needs to be a true/false string. @@ -326,4 +326,4 @@ In addition to the required permissions above, the `s3:PutObjectTagging` permiss ### Contributing Please make all pull requests to the `master` branch and ensure tests pass -locally. +locally. \ No newline at end of file diff --git a/go.mod b/go.mod index e58ad426..45fb62b8 100644 --- a/go.mod +++ b/go.mod @@ -5,54 +5,59 @@ go 1.24.0 toolchain go1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.37.0 - github.com/aws/aws-sdk-go-v2/config v1.30.1 - github.com/aws/aws-sdk-go-v2/credentials v1.18.1 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.1 - github.com/aws/aws-sdk-go-v2/service/s3 v1.85.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.35.0 + github.com/aws/aws-sdk-go-v2 v1.40.0 + github.com/aws/aws-sdk-go-v2/config v1.32.1 + github.com/aws/aws-sdk-go-v2/credentials v1.19.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11 + github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4 github.com/fatih/color v1.18.0 github.com/google/uuid v1.6.0 github.com/h2non/filetype v1.1.3 - github.com/maxbrunsfeld/counterfeiter/v6 v6.11.3 + github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db - github.com/onsi/ginkgo/v2 v2.23.3 - github.com/onsi/gomega v1.37.0 - github.com/vbauerster/mpb/v8 v8.10.2 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 + github.com/vbauerster/mpb/v8 v8.11.2 ) require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.26.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.0 // indirect - github.com/aws/smithy-go v1.22.5 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect + github.com/aws/smithy-go v1.23.2 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect github.com/onsi/ginkgo v1.2.1-0.20170102031522-a23f924ce96d // indirect github.com/rivo/uniseg v0.4.7 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect - golang.org/x/tools v0.35.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.39.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f9de7960..28512730 100644 --- a/go.sum +++ b/go.sum @@ -1,45 +1,91 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/aws/aws-sdk-go-v2 v1.37.0 h1:YtCOESR/pN4j5oA7cVHSfOwIcuh/KwHC4DOSXFbv5F0= github.com/aws/aws-sdk-go-v2 v1.37.0/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= +github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= github.com/aws/aws-sdk-go-v2/config v1.30.1 h1:sHL8g/+9tcZATeV2tEkEfxZeaNokDtKsSjGMGHD49qA= github.com/aws/aws-sdk-go-v2/config v1.30.1/go.mod h1:wkibEyFfxXRyTSzRU4bbF5IUsSXyE4xQ4ZjkGmi5tFo= +github.com/aws/aws-sdk-go-v2/config v1.32.1 h1:iODUDLgk3q8/flEC7ymhmxjfoAnBDwEEYEVyKZ9mzjU= +github.com/aws/aws-sdk-go-v2/config v1.32.1/go.mod h1:xoAgo17AGrPpJBSLg81W+ikM0cpOZG8ad04T2r+d5P0= github.com/aws/aws-sdk-go-v2/credentials v1.18.1 h1:E55xvOqlX7CvB66Z7rSM9usCrFU1ryUIUHqiXsEzVoE= github.com/aws/aws-sdk-go-v2/credentials v1.18.1/go.mod h1:iobSQfR5MkvILxssGOvi/P1jjOhrRzfTiCPCzku0vx4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.1 h1:JeW+EwmtTE0yXFK8SmklrFh/cGTTXsQJumgMZNlbxfM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.1/go.mod h1:BOoXiStwTF+fT2XufhO0Efssbi1CNIO/ZXpZu87N0pw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.0 h1:9sBTeKQwAvmJUWKIACIoiFSnxxl+sS++YDfr17/ngq0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.0/go.mod h1:LW9/PxQD1SYFC7pnWcgqPhoyZprhjEdg5hBK6qYPLW8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.1 h1:aAjA/1JZ1xH/F9XvxJu9cXZyp7BqVXtHpztyZ6p799E= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.1/go.mod h1:ia9HASsPba/7o84sp6iE4wZPdNTJ0oicBe5sWQQk+Ys= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11 h1:NMchKj9gGzIJH4yln7g+Ci4BeVSCayE8CQ7cc+xH9FM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11/go.mod h1:eTZ6Kj2kFJ7UkKEWjlRPYI3fKcH+jKnsSaIom2XABBQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.0 h1:H2iZoqW/v2Jnrh1FnU725Bq6KJ0k2uP63yH+DcY+HUI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.0/go.mod h1:L0FqLbwMXHvNC/7crWV1iIxUlOKYZUE8KuTIA+TozAI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.0 h1:EDped/rNzAhFPhVY0sDGbtD16OKqksfA8OjF/kLEgw8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.0/go.mod h1:uUI335jvzpZRPpjYx6ODc/wg1qH+NnoSTK/FwVeK0C0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.0 h1:iLvW/zOkHGU3BDU5thWnj+UZ9pjhuVhv1loLj7yVtBw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.0/go.mod h1:Fn3gvhdF1x5Rs9nUoCy/fJT1ms8f8dO7RqM9lJHuazQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.0 h1:qGyLBQPphYzUf+IIlb5tHnvg1U2Vc5hXPcP7oRSQfy0= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.0/go.mod h1:g+dzKSLXiR/8ATkPXmLhPOI6rDdjLP3tngeo3FvDcIw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.0 h1:eRhU3Sh8dGbaniI6B+I48XJMrTPRkK4DKo+vqIxziOU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.0/go.mod h1:paNLV18DZ6FnWE/bd06RIKPDIFpjuvCkGKWTG/GDBeM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.0 h1:6jusT+XCcvnD+Elxvm7bUf5sCMTpZEp3AKjYQ4tWJSo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.0/go.mod h1:LimGpdIF/sTBdgqwOEkrArXLCoTamK/9L9x8IKBFTIc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= github.com/aws/aws-sdk-go-v2/service/s3 v1.85.0 h1:gAV4NEp4A+JOrIdoXkAeyy6IOo7+X2s/jRuaHKYiMaU= github.com/aws/aws-sdk-go-v2/service/s3 v1.85.0/go.mod h1:JIQwK8sZ5MuKGm5rrFwp9MHUcyYEsQNpVixuPDlnwaU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 h1:8FshVvnV2sr9kOSAbOnc/vwVmmAwMjOedKH6JW2ddPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 h1:BDgIUYGEo5TkayOWv/oBLPphWwNm/A91AebUjAu5L5g= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.1/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= github.com/aws/aws-sdk-go-v2/service/sso v1.26.0 h1:cuFWHH87GP1NBGXXfMicUbE7Oty5KpPxN6w4JpmuxYc= github.com/aws/aws-sdk-go-v2/service/sso v1.26.0/go.mod h1:aJBemdlbCKyOXEXdXBqS7E+8S9XTDcOTaoOjtng54hA= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 h1:U//SlnkE1wOQiIImxzdY5PXat4Wq+8rlfVEw4Y7J8as= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.4/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.0 h1:t2va+wewPOYIqC6XyJ4MGjiGKkczMAPsgq5W4FtL9ME= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.0/go.mod h1:ExCTcqYqN0hYYRsDlBVU8+68grqlWdgX9/nZJwQW4aY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 h1:LU8S9W/mPDAU9q0FjCLi0TrCheLMGwzbRpvUMwYspcA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= github.com/aws/aws-sdk-go-v2/service/sts v1.35.0 h1:FD9agdG4CeOGS3ORLByJk56YIXDS7mxFpmZyCtpqExc= github.com/aws/aws-sdk-go-v2/service/sts v1.35.0/go.mod h1:NDzDPbBF1xtSTZUMuZx0w3hIfWzcL7X2AQ0Tr9becIQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 h1:GdGmKtG+/Krag7VfyOXV17xjTCz0i9NT+JnqLTOI5nA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.1/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= +github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= +github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4 h1:J+ghqo7ZubTzelkjo9hntpTtP/9lUCWH9icEmAW+B+Q= github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4/go.mod h1:socxpf5+mELPbosI149vWpNlHK6mbfWFxSWOoSndXR8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -49,12 +95,16 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 h1:3DsUAV+VNEQa2CUVLxCY3f87278uWfIDhJnbdvDjvmE= +github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -69,16 +119,24 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.3 h1:Eaq36EIyJNp7b3qDhjV7jmDVq/yPeW2v4pTqzGbOGB4= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.3/go.mod h1:6KKUoQBZBW6PDXJtNfqeEjPXMj/ITTk+cWK9t9uS5+E= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0 h1:aOeI7xAOVdK+R6xbVsZuU9HmCZYmQVmZgPf9xJUd2Sg= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0/go.mod h1:0hZWbtfeCYUQeAQdPLUzETiBhUSns7O6LDj9vH88xKA= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/onsi/ginkgo v1.2.1-0.20170102031522-a23f924ce96d h1:KFDjWk1pKKFlfIOu/D7DqWtZpoNdrNelD7Yktk5qsag= github.com/onsi/ginkgo v1.2.1-0.20170102031522-a23f924ce96d/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.23.3 h1:edHxnszytJ4lD9D5Jjc4tiDkPBZ3siDeJJkUZJJVkp0= github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -93,19 +151,35 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/vbauerster/mpb/v8 v8.10.2 h1:2uBykSHAYHekE11YvJhKxYmLATKHAGorZwFlyNw4hHM= github.com/vbauerster/mpb/v8 v8.10.2/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= +github.com/vbauerster/mpb/v8 v8.11.2 h1:OqLoHznUVU7SKS/WV+1dB5/hm20YLheYupiHhL5+M1Y= +github.com/vbauerster/mpb/v8 v8.11.2/go.mod h1:mEB/M353al1a7wMUNtiymmPsEkGlJgeJmtlbY5adCJ8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/in/archive.go b/in/archive.go index d34063eb..7781633b 100644 --- a/in/archive.go +++ b/in/archive.go @@ -16,6 +16,8 @@ var archiveMimetypes = []string{ "application/gzip", "application/x-tar", "application/zip", + "application/x-bzip2", + "application/x-bzip", } func mimetype(r *bufio.Reader) (string, error) { @@ -68,6 +70,9 @@ func inflate(mime, path, destination string) error { case "application/gzip", "application/x-gzip": cmd = exec.Command("gunzip", path) + case "application/x-bzip2", "application/x-bzip": + cmd = exec.Command("bunzip2", path) + default: return fmt.Errorf("don't know how to extract %s", mime) } diff --git a/in/command.go b/in/command.go index 877ebf53..a283ee84 100644 --- a/in/command.go +++ b/in/command.go @@ -244,14 +244,22 @@ func extractArchive(mime, filename string) error { return fmt.Errorf("failed to extract archive: %s", err) } - if mime == "application/gzip" || mime == "application/x-gzip" { + // Special handling for gzip and bzip2: check if there's a tar inside + if mime == "application/gzip" || mime == "application/x-gzip" || + mime == "application/x-bzip2" || mime == "application/x-bzip" { + + compressionType := "gzip" + if mime == "application/x-bzip2" || mime == "application/x-bzip" { + compressionType = "bzip2" + } + fileInfos, err := os.ReadDir(destDir) if err != nil { return fmt.Errorf("failed to read dir: %s", err) } if len(fileInfos) != 1 { - return fmt.Errorf("%d files found after gunzip; expected 1", len(fileInfos)) + return fmt.Errorf("%d files found after %s decompression; expected 1", len(fileInfos), compressionType) } filename = filepath.Join(destDir, fileInfos[0].Name()) diff --git a/in/command_test.go b/in/command_test.go index 66e104dd..ad0844a9 100644 --- a/in/command_test.go +++ b/in/command_test.go @@ -7,6 +7,7 @@ import ( "io" "log" "os" + "os/exec" "path" "path/filepath" "strings" @@ -417,6 +418,101 @@ var _ = Describe("In Command", func() { }) }) + Context("when the file is bzip2 compressed", func() { + BeforeEach(func() { + request.Version.Path = "files/a-file-1.3.bz2" + request.Source.Regexp = "files/a-file-(.*).bz2" + + s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { + // Create uncompressed file + uncompressedPath := filepath.Join(tmpPath, "uncompressed-file") + err := os.WriteFile(uncompressedPath, []byte("some-contents"), os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + // Compress with bzip2 command + cmd := exec.Command("bzip2", "-c", uncompressedPath) + compressed, err := cmd.Output() + Expect(err).NotTo(HaveOccurred()) + + // Write compressed data to localPath + err = os.WriteFile(localPath, compressed, os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + return nil + } + }) + + It("decompresses the bzip2 file", func() { + _, err := command.Run(destDir, request) + Expect(err).NotTo(HaveOccurred()) + + bs, err := os.ReadFile(filepath.Join(destDir, "a-file-1.3")) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(bs)).To(Equal("some-contents")) + }) + }) + + Context("when the file is a bzip2 compressed tarball", func() { + BeforeEach(func() { + request.Version.Path = "files/a-file-1.3.tar.bz2" + request.Source.Regexp = "files/a-file-(.*).tar.bz2" + + s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { + // Create directory structure + err := os.MkdirAll(filepath.Join(tmpPath, "some-dir"), os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + someFile1 := filepath.Join(tmpPath, "some-dir", "some-file") + err = os.WriteFile(someFile1, []byte("some-contents"), os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + someFile2 := filepath.Join(tmpPath, "some-file") + err = os.WriteFile(someFile2, []byte("some-other-contents"), os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + // Create tarball + tarPath := filepath.Join(tmpPath, "some-tar") + err = createTarball([]string{someFile1, someFile2}, tmpPath, tarPath) + Expect(err).NotTo(HaveOccurred()) + + _, err = os.Stat(tarPath) + Expect(err).NotTo(HaveOccurred()) + + // Read the tar file + tarData, err := os.ReadFile(tarPath) + Expect(err).NotTo(HaveOccurred()) + + // Compress with bzip2 command + cmd := exec.Command("bzip2", "-c") + cmd.Stdin = strings.NewReader(string(tarData)) + compressed, err := cmd.Output() + Expect(err).NotTo(HaveOccurred()) + + // Write compressed tar to localPath + err = os.WriteFile(localPath, compressed, os.ModePerm) + Expect(err).NotTo(HaveOccurred()) + + return nil + } + }) + + It("extracts the bzip2 compressed tarball", func() { + _, err := command.Run(destDir, request) + Expect(err).NotTo(HaveOccurred()) + + Expect(filepath.Join(destDir, "some-dir", "some-file")).To(BeARegularFile()) + + bs, err := os.ReadFile(filepath.Join(destDir, "some-dir", "some-file")) + Expect(err).NotTo(HaveOccurred()) + Expect(bs).To(Equal([]byte("some-contents"))) + + bs, err = os.ReadFile(filepath.Join(destDir, "some-file")) + Expect(err).NotTo(HaveOccurred()) + Expect(bs).To(Equal([]byte("some-other-contents"))) + }) + }) + Context("when the file is not an archive", func() { BeforeEach(func() { s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { diff --git a/integration/in_test.go b/integration/in_test.go index 5b366d15..28ab4139 100644 --- a/integration/in_test.go +++ b/integration/in_test.go @@ -402,4 +402,226 @@ var _ = Describe("in", func() { Ω(actualTagsJSON).Should(MatchJSON(expectedTagsJSON)) }) }) + + Context("when unpack is true with a .tar.bz2 file", func() { + var directoryPrefix string + + BeforeEach(func() { + directoryPrefix = "in-request-unpack-bz2" + inRequest = in.Request{ + Source: s3resource.Source{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + SessionToken: sessionToken, + AwsRoleARN: awsRoleARN, + Bucket: bucketName, + RegionName: regionName, + Endpoint: endpoint, + Regexp: filepath.Join(directoryPrefix, "archive-(.*)"), + UsePathStyle: pathStyle, + }, + Version: s3resource.Version{ + Path: filepath.Join(directoryPrefix, "archive-1.tar.bz2"), + }, + Params: in.Params{ + Unpack: true, + }, + } + + // Create temp directory for building the archive + archiveTempDir, err := os.MkdirTemp("", "archive-build") + Ω(err).ShouldNot(HaveOccurred()) + defer os.RemoveAll(archiveTempDir) + + // Create test files + err = os.WriteFile(filepath.Join(archiveTempDir, "file1.txt"), []byte("content1"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + err = os.WriteFile(filepath.Join(archiveTempDir, "file2.txt"), []byte("content2"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(filepath.Join(archiveTempDir, "subdir"), 0755) + Ω(err).ShouldNot(HaveOccurred()) + err = os.WriteFile(filepath.Join(archiveTempDir, "subdir", "file3.txt"), []byte("content3"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + + // Create tar.bz2 archive + archiveFile := filepath.Join(archiveTempDir, "archive-1.tar.bz2") + tarCmd := exec.Command("tar", "-cjf", archiveFile, "-C", archiveTempDir, "file1.txt", "file2.txt", "subdir") + err = tarCmd.Run() + Ω(err).ShouldNot(HaveOccurred()) + + // Upload to S3 + _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, "archive-1.tar.bz2"), archiveFile, s3resource.NewUploadFileOptions()) + Ω(err).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + err := s3client.DeleteFile(bucketName, filepath.Join(directoryPrefix, "archive-1.tar.bz2")) + Ω(err).ShouldNot(HaveOccurred()) + }) + + It("extracts the archive contents", func() { + // Verify the archive file was removed after extraction + Ω(filepath.Join(destDir, "archive-1.tar.bz2")).ShouldNot(BeARegularFile()) + + // Verify all files were extracted + Ω(filepath.Join(destDir, "file1.txt")).Should(BeARegularFile()) + content1, err := os.ReadFile(filepath.Join(destDir, "file1.txt")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content1).Should(Equal([]byte("content1"))) + + Ω(filepath.Join(destDir, "file2.txt")).Should(BeARegularFile()) + content2, err := os.ReadFile(filepath.Join(destDir, "file2.txt")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content2).Should(Equal([]byte("content2"))) + + Ω(filepath.Join(destDir, "subdir", "file3.txt")).Should(BeARegularFile()) + content3, err := os.ReadFile(filepath.Join(destDir, "subdir", "file3.txt")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content3).Should(Equal([]byte("content3"))) + + // Verify version file is created + Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(versionContents).Should(Equal([]byte("1"))) + }) + }) + + Context("when unpack is true with a .tar.gz file", func() { + var directoryPrefix string + + BeforeEach(func() { + directoryPrefix = "in-request-unpack-gz" + inRequest = in.Request{ + Source: s3resource.Source{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + SessionToken: sessionToken, + AwsRoleARN: awsRoleARN, + Bucket: bucketName, + RegionName: regionName, + Endpoint: endpoint, + Regexp: filepath.Join(directoryPrefix, "archive-(.*)"), + UsePathStyle: pathStyle, + }, + Version: s3resource.Version{ + Path: filepath.Join(directoryPrefix, "archive-1.tar.gz"), + }, + Params: in.Params{ + Unpack: true, + }, + } + + // Create temp directory for building the archive + archiveTempDir, err := os.MkdirTemp("", "archive-build-gz") + Ω(err).ShouldNot(HaveOccurred()) + defer os.RemoveAll(archiveTempDir) + + // Create test files + err = os.WriteFile(filepath.Join(archiveTempDir, "gzfile1.txt"), []byte("gz-content1"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + err = os.WriteFile(filepath.Join(archiveTempDir, "gzfile2.txt"), []byte("gz-content2"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + + // Create tar.gz archive + archiveFile := filepath.Join(archiveTempDir, "archive-1.tar.gz") + tarCmd := exec.Command("tar", "-czf", archiveFile, "-C", archiveTempDir, "gzfile1.txt", "gzfile2.txt") + err = tarCmd.Run() + Ω(err).ShouldNot(HaveOccurred()) + + // Upload to S3 + _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, "archive-1.tar.gz"), archiveFile, s3resource.NewUploadFileOptions()) + Ω(err).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + err := s3client.DeleteFile(bucketName, filepath.Join(directoryPrefix, "archive-1.tar.gz")) + Ω(err).ShouldNot(HaveOccurred()) + }) + + It("extracts the archive contents (regression test)", func() { + // Verify the archive file was removed after extraction + Ω(filepath.Join(destDir, "archive-1.tar.gz")).ShouldNot(BeARegularFile()) + + // Verify files were extracted + Ω(filepath.Join(destDir, "gzfile1.txt")).Should(BeARegularFile()) + content1, err := os.ReadFile(filepath.Join(destDir, "gzfile1.txt")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content1).Should(Equal([]byte("gz-content1"))) + + Ω(filepath.Join(destDir, "gzfile2.txt")).Should(BeARegularFile()) + content2, err := os.ReadFile(filepath.Join(destDir, "gzfile2.txt")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content2).Should(Equal([]byte("gz-content2"))) + }) + }) + + Context("when unpack is true with a plain .bz2 file", func() { + var directoryPrefix string + + BeforeEach(func() { + directoryPrefix = "in-request-unpack-bz2-plain" + inRequest = in.Request{ + Source: s3resource.Source{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + SessionToken: sessionToken, + AwsRoleARN: awsRoleARN, + Bucket: bucketName, + RegionName: regionName, + Endpoint: endpoint, + Regexp: filepath.Join(directoryPrefix, "file-(.*)"), + UsePathStyle: pathStyle, + }, + Version: s3resource.Version{ + Path: filepath.Join(directoryPrefix, "file-1.bz2"), + }, + Params: in.Params{ + Unpack: true, + }, + } + + // Create temp directory for building the compressed file + compressTempDir, err := os.MkdirTemp("", "compress-build") + Ω(err).ShouldNot(HaveOccurred()) + defer os.RemoveAll(compressTempDir) + + // Create test file + testFile := filepath.Join(compressTempDir, "file-1") + err = os.WriteFile(testFile, []byte("plain bz2 content"), 0644) + Ω(err).ShouldNot(HaveOccurred()) + + // Compress with bzip2 + compressCmd := exec.Command("bzip2", "-k", testFile) + err = compressCmd.Run() + Ω(err).ShouldNot(HaveOccurred()) + + // Upload to S3 + compressedFile := testFile + ".bz2" + _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, "file-1.bz2"), compressedFile, s3resource.NewUploadFileOptions()) + Ω(err).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + err := s3client.DeleteFile(bucketName, filepath.Join(directoryPrefix, "file-1.bz2")) + Ω(err).ShouldNot(HaveOccurred()) + }) + + It("decompresses the file", func() { + // Verify the compressed file was removed after decompression + Ω(filepath.Join(destDir, "file-1.bz2")).ShouldNot(BeARegularFile()) + + // Verify decompressed file exists + Ω(filepath.Join(destDir, "file-1")).Should(BeARegularFile()) + content, err := os.ReadFile(filepath.Join(destDir, "file-1")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(content).Should(Equal([]byte("plain bz2 content"))) + + // Verify version file is created + Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) + Ω(err).ShouldNot(HaveOccurred()) + Ω(versionContents).Should(Equal([]byte("1"))) + }) + }) }) diff --git a/s3client_test.go b/s3client_test.go index f4f82086..d20e86cb 100644 --- a/s3client_test.go +++ b/s3client_test.go @@ -171,6 +171,7 @@ var _ = Describe("S3Resource", func() { false, true, true, + "", ) })