Add WARC format regression smoke test

Introduce TestSmokeWARCFormatRegression to validate WARC format consistency using a frozen reference file (testdata/test.warc.gz). This test checks exact byte counts, record counts, Content-Length values, and digest hashes against known-good values.

This complements the existing dynamic tests by providing explicit validation that the WARC format hasn't changed, addressing the concern about byte-level format regression detection while keeping the main integration tests maintainable.

Author: CorentinB

smoke_test.go

@@ -0,0 +1,154 @@
+package warc
+
+import (
+	"io"
+	"os"
+	"strconv"
+	"testing"
+)
+
+// TestSmokeWARCFormatRegression validates that the WARC format remains consistent
+// by checking a frozen reference file (testdata/test.warc.gz) against known-good values.
+//
+// This test serves as a regression detector for WARC format changes, complementing the
+// dynamic tests in client_test.go. It addresses the concern that byte-level format
+// changes should be explicitly validated against a known-good snapshot.
+//
+// If this test fails, it indicates that either:
+// 1. The WARC writing logic has changed in a way that affects the format
+// 2. The reference file has been modified
+// 3. There's a bug in the record serialization
+func TestSmokeWARCFormatRegression(t *testing.T) {
+	const testFile = "testdata/test.warc.gz"
+
+	// Expected file-level metrics
+	const expectedFileSize = 22350 // bytes (compressed)
+	const expectedTotalRecords = 3
+	const expectedTotalContentLength = 22083 // sum of all Content-Length values
+
+	// Expected record-level metrics
+	// These values were extracted from a known-good WARC file and serve as
+	// a snapshot of correct format behavior.
+	expectedRecords := []struct {
+		warcType        string
+		contentLength   int64
+		blockDigest     string
+		payloadDigest   string // only for response records
+		targetURI       string // only for response records
+	}{
+		{
+			warcType:      "warcinfo",
+			contentLength: 143,
+			blockDigest:   "sha1:IYWIATZSPEOF7U5W7VGGJOSQTIWUDXQ6",
+		},
+		{
+			warcType:      "request",
+			contentLength: 110,
+			blockDigest:   "sha1:JNDMG56JVTVVOQSDQRD25XWTGMRQAQDB",
+		},
+		{
+			warcType:      "response",
+			contentLength: 21830,
+			blockDigest:   "sha1:LCKC4TTRSBWYHGYT5P22ON4DWY65WHDZ",
+			targetURI:     "https://apis.google.com/js/platform.js",
+		},
+	}
+
+	// Validate file size
+	stat, err := os.Stat(testFile)
+	if err != nil {
+		t.Fatalf("failed to stat test file: %v", err)
+	}
+	if stat.Size() != expectedFileSize {
+		t.Errorf("file size mismatch: expected %d bytes, got %d bytes", expectedFileSize, stat.Size())
+	}
+
+	// Open and read WARC file
+	file, err := os.Open(testFile)
+	if err != nil {
+		t.Fatalf("failed to open test file: %v", err)
+	}
+	defer file.Close()
+
+	reader, err := NewReader(file)
+	if err != nil {
+		t.Fatalf("failed to create WARC reader: %v", err)
+	}
+
+	var recordCount int
+	var totalContentLength int64
+
+	// Read and validate each record
+	for recordCount < expectedTotalRecords {
+		record, _, err := reader.ReadRecord()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Fatalf("failed to read record %d: %v", recordCount+1, err)
+		}
+		if record == nil {
+			break
+		}
+
+		expected := expectedRecords[recordCount]
+
+		// Validate WARC-Type
+		warcType := record.Header.Get("WARC-Type")
+		if warcType != expected.warcType {
+			t.Errorf("record %d: WARC-Type mismatch: expected %q, got %q",
+				recordCount+1, expected.warcType, warcType)
+		}
+
+		// Validate Content-Length
+		contentLengthStr := record.Header.Get("Content-Length")
+		contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
+		if err != nil {
+			t.Errorf("record %d: failed to parse Content-Length %q: %v",
+				recordCount+1, contentLengthStr, err)
+		} else {
+			if contentLength != expected.contentLength {
+				t.Errorf("record %d: Content-Length mismatch: expected %d, got %d",
+					recordCount+1, expected.contentLength, contentLength)
+			}
+			totalContentLength += contentLength
+		}
+
+		// Validate WARC-Block-Digest
+		blockDigest := record.Header.Get("WARC-Block-Digest")
+		if blockDigest != expected.blockDigest {
+			t.Errorf("record %d: WARC-Block-Digest mismatch: expected %q, got %q",
+				recordCount+1, expected.blockDigest, blockDigest)
+		}
+
+		// Validate response-specific fields
+		if warcType == "response" {
+			if expected.targetURI != "" {
+				targetURI := record.Header.Get("WARC-Target-URI")
+				if targetURI != expected.targetURI {
+					t.Errorf("record %d: WARC-Target-URI mismatch: expected %q, got %q",
+						recordCount+1, expected.targetURI, targetURI)
+				}
+			}
+		}
+
+		// Close record content
+		if err := record.Content.Close(); err != nil {
+			t.Errorf("record %d: failed to close content: %v", recordCount+1, err)
+		}
+
+		recordCount++
+	}
+
+	// Validate total record count
+	if recordCount != expectedTotalRecords {
+		t.Errorf("total record count mismatch: expected %d, got %d",
+			expectedTotalRecords, recordCount)
+	}
+
+	// Validate total content length
+	if totalContentLength != expectedTotalContentLength {
+		t.Errorf("total content length mismatch: expected %d bytes, got %d bytes",
+			expectedTotalContentLength, totalContentLength)
+	}
+}