Add FIO tests to check the data integrity

azure-pipeline-templates/fio-data-integrity.yml

@@ -0,0 +1,119 @@
+# This File emulates various workloads using fio utility
+# to check the data integrity. If you are adding any tests make sure
+# to add integrity validation step in fio job file/command.
+#
+parameters:
+  - name: config_file
+    type: string
+  - name: cache_mode
+    type: string
+  - name: account_name
+    type: string
+  - name: account_key
+    type: string
+  - name: account_type
+    type: string
+  - name: verbose_log
+    type: boolean
+    default: false
+
+steps:
+    # Generate config file for file cache
+  - ${{ if eq(parameters.cache_mode, 'file_cache') }}:
+    - script: |
+        $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }}
+        cat ${{ parameters.config_file }}
+      displayName: 'Create Config File for File Cache'
+      env:
+        STO_ACC_NAME: ${{ parameters.account_name }}
+        STO_ACC_KEY: ${{ parameters.account_key }}
+        STO_ACC_TYPE: ${{ parameters.account_type }}
+        VERBOSE_LOG: ${{ parameters.verbose_log }}
+      continueOnError: false
+
+  # Generate Config file for block cache
+  - ${{ if eq(parameters.cache_mode, 'block_cache') }}:
+    - script: |
+        $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }}
+        cat ${{ parameters.config_file }}
+      displayName: 'Create Config File for Block Cache'
+      env:
+        STO_ACC_NAME: ${{ parameters.account_name }}
+        STO_ACC_KEY: ${{ parameters.account_key }}
+        STO_ACC_TYPE: ${{ parameters.account_type }}
+        VERBOSE_LOG: ${{ parameters.verbose_log }}
+
+
+  - template: 'mount.yml'
+    parameters:
+      prefix: ${{ parameters.cache_mode }}
+      mountStep:
+        script: |
+          $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200
+
+  - script:
+      sudo apt-get install fio -y
+    displayName: 'Install Utils'
+
+# Run fio on 1 file 10G size SEQ read,write
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING******"
+      fio --rw=readwrite --bs=1M --size=10g --numjobs=1 $(WORK_DIR)/test/fio/rw.fio
+    displayName: 'SEQ-[RW]-10G-1F-1T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on 10 files 1G size SEQ read, write
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio --rw=readwrite --bs=1M --size=1g --numjobs=10 $(WORK_DIR)/test/fio/rw.fio
+    displayName: 'SEQ-[RW]-1G-10F-10T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on 1 file 10GB size RAND read, write
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio --rw=randwrite --bs=1M --size=10g --numjobs=1 $(WORK_DIR)/test/fio/rw.fio
+    displayName: 'RAND-[RW]-10G-1F-1T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on 10 files 1000M size RAND read, write
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio --rw=randwrite --bs=1M --size=1g --numjobs=10 $(WORK_DIR)/test/fio/rw.fio
+    displayName: 'RAND-[RW]-1G-10F-10T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on 1 file 10G size SEQ write, with 10 threads from different offsets(1, 1G, 2G,..)
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio $(WORK_DIR)/test/fio/seq-write-1f-10th.fio
+    displayName: 'SEQ-[W]-10G-1F-10T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on file having hole inside blocks.(hole can be at middle/at the edges)
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio $(WORK_DIR)/test/fio/hole_inside_blocks.fio
+    displayName: 'SEQ-HOLE3M-10G-1F-1T'
+    workingDirectory: $(MOUNT_DIR)
+
+# Run fio on file having hole over blocks.(whole blocks can be of holes)
+  - script: |
+      ls -lrt
+      rm -rf ./*
+      echo "******FIO JOB STARTING*******"
+      fio $(WORK_DIR)/test/fio/hole_over_blocks.fio
+    displayName: 'SEQ-HOLE16M-10G-1F-1T'
+    workingDirectory: $(MOUNT_DIR)

blobfuse2-nightly.yaml

@@ -31,6 +31,11 @@ parameters:
     type: boolean
     default: true
 
+  - name: fio_test
+    displayName: 'Run fio workloads to check data integrity'
+    type: boolean
+    default: false
+
   - name: linux_git_test
     displayName: 'Compile Linux Source and Run git clone Tests'
     type: string
@@ -1069,6 +1074,83 @@ stages:
                 unmount: true
                 delete_containers: true
 
+  - ${{ if eq(parameters.fio_test, true) }}:
+    - stage: FioTests
+      dependsOn: []
+      jobs:
+        # Ubuntu Tests
+        - job: Set_1
+          timeoutInMinutes: 60
+          strategy:
+            matrix:
+              Ubuntu-20:
+                AgentName: 'blobfuse-ubuntu20'
+                distro: 'ubuntu'
+                distro_version: 'ubuntu_20_x86'
+                fuselib: 'libfuse-dev'
+                tags: 'fuse2'
+              Ubuntu-22:
+                AgentName: 'blobfuse-ubuntu22'
+                distro: 'ubuntu'
+                distro_version: 'ubuntu_22_x86'
+                fuselib: 'libfuse3-dev'
+                tags: 'fuse3'
+
+          pool:
+            name: "blobfuse-ubuntu-pool"
+            demands:
+              - ImageOverride -equals $(AgentName)
+
+          variables:
+            - group: NightlyBlobFuse
+            - name: MOUNT_DIR
+              value: '$(Pipeline.Workspace)/blob_mnt'
+            - name: TEMP_DIR
+              value: '$(Pipeline.Workspace)/blobfuse2_tmp'
+            - name: BLOBFUSE2_CFG
+              value: '$(Pipeline.Workspace)/blobfuse2.yaml'
+            - name: BLOBFUSE2_ADLS_CFG
+              value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
+            - name: skipComponentGovernanceDetection
+              value: true
+            - name: GOPATH
+              value: '$(Pipeline.Workspace)/go'
+            - name: ROOT_DIR
+              value: '$(Agent.TempDirectory)'
+            - name: WORK_DIR
+              value: '$(Build.SourcesDirectory)'
+
+          steps:
+            # -------------------------------------------------------
+            # Pull and build the code
+            - template: 'azure-pipeline-templates/build.yml'
+              parameters:
+                skip_ut: true
+
+            - template: 'azure-pipeline-templates/fio-data-integrity.yml'
+              parameters:
+                cache_mode: file_cache
+                config_file: $(BLOBFUSE2_CFG)
+                account_name: $(BF2_BLK_ACC_NAME)
+                account_key: $(BF2_BLK_ACC_KEY)
+                account_type: block
+                verbose_log: ${{ parameters.verbose_log }}
+
+            - template: 'azure-pipeline-templates/fio-data-integrity.yml'
+              parameters:
+                cache_mode: block_cache
+                config_file: $(BLOBFUSE2_CFG)
+                account_name: $(BF2_BLK_ACC_NAME)
+                account_key: $(BF2_BLK_ACC_KEY)
+                account_type: block
+                verbose_log: ${{ parameters.verbose_log }}
+
+            - template: 'azure-pipeline-templates/cleanup.yml'
+              parameters:
+                unmount: true
+                delete_containers: true
+
+
   - ${{ if ne(parameters.linux_git_test, 'none') }}:
     - stage: CompileLinux_GitClone
       dependsOn: []

test/fio/hole_inside_blocks.fio

@@ -0,0 +1,18 @@
+[global]
+name=blobfuse_file_integrity
+bs=1M
+filename=hole_inside_blocks_fio.data
+verify=crc32c
+verify_fatal=1
+verify_dump=1
+group_reporting
+
+[hole]
+size=10G
+# Create 3M Hole after each write call.
+# This will create the holes inside the blocks(assuming block size of 8M)
+rw=write:3M
+number_ios=2560
+ioengine=sync
+fallocate=none
+unlink=1

test/fio/hole_over_blocks.fio

@@ -0,0 +1,18 @@
+[global]
+name=blobfuse_file_integrity
+bs=1M
+filename=hole_over_blocks_fio.data
+verify=crc32c
+verify_fatal=1
+group_reporting
+
+[hole]
+size=10G
+# Create 16M Hole/2 blocks after each write call.
+# This will create the holes over the blocks(assuming block size of 8M), where some blocks will be
+# skipped and other blocks to have some null data at the end of the block.
+rw=write:15M
+number_ios=640
+ioengine=sync
+fallocate=none
+unlink=1

test/fio/rw.fio

@@ -0,0 +1,7 @@
+[writers]
+bs=1M
+ioengine=sync
+verify=crc32c
+verify_fatal=1
+group_reporting
+unlink=1

test/fio/seq-write-1f-10th.fio

@@ -0,0 +1,78 @@
+[global]
+name=blobfuse_file_integrity
+bs=1M
+size=10G
+filename=stripe_write_fio.data
+verify=crc32c
+verify_fatal=1
+ioengine=sync
+group_reporting
+
+[job1]
+offset=0
+size=1G
+rw=write
+fallocate=none
+
+
+[job2]
+offset=1G
+size=1G
+rw=write
+fallocate=none
+
+
+[job3]
+offset=2G
+size=1G
+rw=write
+fallocate=none
+
+
+[job4]
+offset=3G
+size=1G
+rw=write
+fallocate=none
+
+
+[job5]
+offset=4G
+size=1G
+rw=write
+fallocate=none
+
+
+[job6]
+offset=5G
+size=1G
+rw=write
+fallocate=none
+
+
+[job7]
+offset=6G
+size=1G
+rw=write
+fallocate=none
+
+
+[job8]
+offset=7G
+size=1G
+rw=write
+fallocate=none
+
+
+[job9]
+offset=8G
+size=1G
+rw=write
+fallocate=none
+
+
+[job10]
+offset=9G
+size=1G
+rw=write
+fallocate=none