From 5feffa1155c5d7b9081be0960071f86f889c84ee Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 02:47:50 -0700 Subject: [PATCH 01/60] Updated' --- .github/workflows/aicomment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 54424603d5..de9d6c1122 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | - apt install jq + sudo apt-get install jq python -m pip install --upgrade pip pip install mcp httpx python-dotenv requests openai transformers torch From e9078d2bff558792aa9b34b6dda83c5d648adb48 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 03:15:51 -0700 Subject: [PATCH 02/60] Updated" --- .github/scripts/ai_summarize.py | 14 +++++++++ .github/scripts/extract_text.py | 50 +++++++++++++++++++++++++++++++++ .github/workflows/aicomment.yml | 20 +++++++++++-- 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/extract_text.py diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 827e018271..ef80f53498 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -1,5 +1,6 @@ import sys import os +import subprocess from transformers import pipeline, AutoTokenizer # The maximum sequence length for this model. This is a fixed constraint. @@ -94,6 +95,19 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): resp_len = len(full_text) if resp_len < 65000: # Responce was smaller so just use the full text + result = subprocess.run( + ["python", ".github/scripts/extract_text.py", input_file_path], + capture_output=True, + text=True + ) + + # The result object contains the output + if result.returncode == 0: + full_text = result.stdout + else: + print(f"\n--- Script Error (stderr) ---") + print(result.stderr) + summary = full_text else: # Use the LLM to summarize the text diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py new file mode 100644 index 0000000000..08d57b1297 --- /dev/null +++ b/.github/scripts/extract_text.py @@ -0,0 +1,50 @@ +import re +import sys + +def extract_text_field(text_content: str) -> str: + """ + Extracts the 'text' field from a string representation of a TextContent object. + + Args: + text_content: The input string containing the TextContent representation. + + Returns: + The extracted text field as a string, or an empty string if not found. + """ + # The new pattern uses a greedy match `(.*)` to capture all characters, + # including newlines (re.DOTALL). The positive lookahead `(?=...)` + # ensures that the match ends exactly before the specific trailing part + # of the string (', annotations=None, meta=None)]'). This prevents + # the regex from stopping at single quotes within the text content. + try: + pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" + match = re.search(pattern, text_content, re.DOTALL) + if match: + return match.group(1) + except re.error as e: + print(f"Regular expression error: {e}") + + return "" + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python extract_text.py ") + sys.exit(1) + + file_path = sys.argv[1] + + try: + with open(file_path, 'r', encoding='utf-8') as f: + input_text = f.read() + + extracted_content = extract_text_field(input_text) + + if extracted_content: + print(extracted_content) + else: + print("Failed to extract the 'text' field.") + + except FileNotFoundError: + print(f"Error: The file '{file_path}' was not found.") + except Exception as e: + print(f"An unexpected error occurred: {e}") diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index de9d6c1122..e5295e96cd 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -55,8 +55,24 @@ jobs: run: cat response_comment.md - name: Post Comment to Issue + if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 with: - issue-number: ${{ github.event.issue.number || github.event.discussion.number }} + issue-number: ${{ github.event.issue.number }} body-path: 'response_comment.md' - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Post Comment to Discussion + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + uses: peter-evans/create-or-update-discussion-comment@v1 + with: + discussion-number: ${{ github.event.discussion.number }} + body-path: 'response_comment.md' + token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Post Comment + # uses: peter-evans/create-or-update-comment@v4 + # with: + # issue-number: ${{ github.event.issue.number || github.event.discussion.number }} + # body-path: 'response_comment.md' + # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From d510befa7c4ff37ee3504ce440424e38e58aa479 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 03:41:31 -0700 Subject: [PATCH 03/60] Updated --- .github/workflows/aicomment.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index e5295e96cd..447e0a4793 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -61,14 +61,24 @@ jobs: issue-number: ${{ github.event.issue.number }} body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read comment body from file + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + id: read_body + run: | + echo "body<> "$GITHUB_OUTPUT" + cat response_comment.md >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - uses: peter-evans/create-or-update-discussion-comment@v1 + uses: wesleyscholl/create-discussion-comment@v1.0.18 with: - discussion-number: ${{ github.event.discussion.number }} - body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} + body: ${{ steps.read_body.outputs.body }} + discussion-id: ${{ github.event.discussion.id }} + client-mutation-id: "unique-id-1234" + # - name: Post Comment # uses: peter-evans/create-or-update-comment@v4 From b0a8444caeb1377da5b4e959c8ff92cd3f5fe3af Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 03:43:17 -0700 Subject: [PATCH 04/60] Updated --- .github/scripts/ai_summarize.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index ef80f53498..0f645826f8 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -91,23 +91,25 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): with open(input_file_path, 'r', encoding='utf-8') as f: full_text = f.read() + + # Call the extract_text.py script to get the cleaned text + result = subprocess.run( + ["python", ".github/scripts/extract_text.py", input_file_path], + capture_output=True, + text=True + ) + # The result object contains the output + if result.returncode == 0: + full_text = result.stdout + else: + print(f"\n--- Script Error (stderr) ---") + print(result.stderr) + resp_len = len(full_text) + if resp_len < 65000: # Responce was smaller so just use the full text - result = subprocess.run( - ["python", ".github/scripts/extract_text.py", input_file_path], - capture_output=True, - text=True - ) - - # The result object contains the output - if result.returncode == 0: - full_text = result.stdout - else: - print(f"\n--- Script Error (stderr) ---") - print(result.stderr) - summary = full_text else: # Use the LLM to summarize the text From 125366cd15cd71fe11822693229fae75ca0e2867 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 03:56:09 -0700 Subject: [PATCH 05/60] Updated --- .github/scripts/ai_summarize.py | 2 +- .github/scripts/extract_text.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 0f645826f8..0772654276 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -120,7 +120,7 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): "### AI Generated Response\n\n" f"{summary}\n\n" "---\n" - "**Kindly share mount command, config file and debug-logs for further investigation.**\n\n" + "**In case of issu, kindly share mount command, config file and debug-logs to investigate further.**\n\n" "---\n" "*Disclaimer: This summary is AI-generated and may not be fully accurate.*" ) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 08d57b1297..0dacca4cf1 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -20,7 +20,7 @@ def extract_text_field(text_content: str) -> str: pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" match = re.search(pattern, text_content, re.DOTALL) if match: - return match.group(1) + return match.group(1).encode('utf-8').decode('unicode_escape') except re.error as e: print(f"Regular expression error: {e}") From 56aaa25976a1cb324ada42cb9cb304a92bb9f0de Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 03:58:13 -0700 Subject: [PATCH 06/60] Updated --- .github/workflows/aicomment.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 447e0a4793..f69445003c 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -54,6 +54,13 @@ jobs: - name: Print summarized response run: cat response_comment.md + - name: Read comment body from file + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + id: read_body + run: | + body=$(cat response_comment.md) + printf "body=%s\n" "$body" >> "$GITHUB_OUTPUT" + - name: Post Comment to Issue if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 @@ -61,14 +68,6 @@ jobs: issue-number: ${{ github.event.issue.number }} body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} - - - name: Read comment body from file - if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - id: read_body - run: | - echo "body<> "$GITHUB_OUTPUT" - cat response_comment.md >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' From cb23eb6284b21802de74d9107d689f6346204875 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:09:08 -0700 Subject: [PATCH 07/60] Updated --- .github/scripts/extract_text.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 0dacca4cf1..d008e2af9a 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -17,7 +17,8 @@ def extract_text_field(text_content: str) -> str: # of the string (', annotations=None, meta=None)]'). This prevents # the regex from stopping at single quotes within the text content. try: - pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" + # pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" + pattern = r"text='(.*)'(?=, Notes:)" match = re.search(pattern, text_content, re.DOTALL) if match: return match.group(1).encode('utf-8').decode('unicode_escape') From 21d1a8e5825e057cfd2ac43eeb88bc8326b8619a Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:15:50 -0700 Subject: [PATCH 08/60] Updated" --- .github/scripts/extract_text.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index d008e2af9a..37af81b181 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -17,11 +17,12 @@ def extract_text_field(text_content: str) -> str: # of the string (', annotations=None, meta=None)]'). This prevents # the regex from stopping at single quotes within the text content. try: - # pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" - pattern = r"text='(.*)'(?=, Notes:)" + pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" match = re.search(pattern, text_content, re.DOTALL) if match: - return match.group(1).encode('utf-8').decode('unicode_escape') + decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') + cleaned_text = decoded_text.split("Notes:")[0].strip() + return cleaned_text except re.error as e: print(f"Regular expression error: {e}") From cbd4aa4a7795d1dcfb726da06a3f74891ed28381 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:27:13 -0700 Subject: [PATCH 09/60] Updated --- .github/scripts/ai_summarize.py | 2 +- .github/workflows/aicomment.yml | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 0772654276..721df3e943 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -120,7 +120,7 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): "### AI Generated Response\n\n" f"{summary}\n\n" "---\n" - "**In case of issu, kindly share mount command, config file and debug-logs to investigate further.**\n\n" + "**In case of issue, share mount command, config file and debug-logs to investigate further.**\n\n" "---\n" "*Disclaimer: This summary is AI-generated and may not be fully accurate.*" ) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index f69445003c..56757f9dcb 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -55,11 +55,8 @@ jobs: run: cat response_comment.md - name: Read comment body from file - if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - id: read_body - run: | - body=$(cat response_comment.md) - printf "body=%s\n" "$body" >> "$GITHUB_OUTPUT" + id: get-body + run: echo "COMMENT_BODY=$(cat response_comment.md)" >> $GITHUB_ENV - name: Post Comment to Issue if: github.event_name == 'issues' @@ -74,7 +71,7 @@ jobs: uses: wesleyscholl/create-discussion-comment@v1.0.18 with: token: ${{ secrets.GITHUB_TOKEN }} - body: ${{ steps.read_body.outputs.body }} + body: ${{ env.COMMENT_BODY }} discussion-id: ${{ github.event.discussion.id }} client-mutation-id: "unique-id-1234" From b0a023a5ddb775b9ac2e5a60a463176613b10b26 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:38:46 -0700 Subject: [PATCH 10/60] Updated --- .github/scripts/extract_text.py | 1 + .github/workflows/aicomment.yml | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 37af81b181..13dd89ab4d 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -22,6 +22,7 @@ def extract_text_field(text_content: str) -> str: if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') cleaned_text = decoded_text.split("Notes:")[0].strip() + cleaned_text = cleaned_text.split("View this search on")[0].strip() return cleaned_text except re.error as e: print(f"Regular expression error: {e}") diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 56757f9dcb..1f2cb24204 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -56,7 +56,12 @@ jobs: - name: Read comment body from file id: get-body - run: echo "COMMENT_BODY=$(cat response_comment.md)" >> $GITHUB_ENV + run: | + # Use a delimiter to correctly handle multi-line strings with GITHUB_OUTPUT + content=$(cat response_comment.md) + echo "COMMENT_BODY<> $GITHUB_OUTPUT + echo "$content" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Post Comment to Issue if: github.event_name == 'issues' @@ -71,7 +76,7 @@ jobs: uses: wesleyscholl/create-discussion-comment@v1.0.18 with: token: ${{ secrets.GITHUB_TOKEN }} - body: ${{ env.COMMENT_BODY }} + body: ${{ steps.get-body.outputs.COMMENT_BODY }} discussion-id: ${{ github.event.discussion.id }} client-mutation-id: "unique-id-1234" From 506e36ff06727143d6953630fd1114d1940206e2 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:43:31 -0700 Subject: [PATCH 11/60] Updated --- .github/scripts/ai_summarize.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 721df3e943..7e91fe8a81 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -108,17 +108,19 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): resp_len = len(full_text) - if resp_len < 65000: - # Responce was smaller so just use the full text - summary = full_text - else: - # Use the LLM to summarize the text - summary = summarize_text_with_llm(full_text) + # Use the LLM to summarize the text + summary = summarize_text_with_llm(full_text) + + if resp_len > 65000: + full_text = "AI Generated response skipped due to large size." # Add the disclaimer final_comment = ( "### AI Generated Response\n\n" + "**Summary**\n\n" f"{summary}\n\n" + "**Details**\n\n" + f"{full_text}\n\n" "---\n" "**In case of issue, share mount command, config file and debug-logs to investigate further.**\n\n" "---\n" From 1cc4bca9a621a589ad3d500868182a57e701209f Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 04:47:55 -0700 Subject: [PATCH 12/60] Updated --- .github/workflows/aicomment.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 1f2cb24204..37c4b97912 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -63,6 +63,9 @@ jobs: echo "$content" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + - name: Debug - Print comment body for discussion + run: echo "${{ steps.get-body.outputs.COMMENT_BODY }}" + - name: Post Comment to Issue if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 @@ -78,12 +81,4 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} body: ${{ steps.get-body.outputs.COMMENT_BODY }} discussion-id: ${{ github.event.discussion.id }} - client-mutation-id: "unique-id-1234" - - - # - name: Post Comment - # uses: peter-evans/create-or-update-comment@v4 - # with: - # issue-number: ${{ github.event.issue.number || github.event.discussion.number }} - # body-path: 'response_comment.md' - # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + client-mutation-id: "unique-id-1234" \ No newline at end of file From 32aab9233d79b7708c45c73ba8f1cefa5647ac1a Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:10:09 -0700 Subject: [PATCH 13/60] Updated --- .github/scripts/ai_summarize.py | 16 ++++++++++------ .github/scripts/extract_text.py | 1 - 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 7e91fe8a81..227e9e22e7 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -111,16 +111,20 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): # Use the LLM to summarize the text summary = summarize_text_with_llm(full_text) - if resp_len > 65000: - full_text = "AI Generated response skipped due to large size." - - # Add the disclaimer final_comment = ( "### AI Generated Response\n\n" "**Summary**\n\n" f"{summary}\n\n" - "**Details**\n\n" - f"{full_text}\n\n" + ) + + if resp_len > 65000: + final_comment += ( + "**Details**\n\n" + f"{full_text}\n\n" + ) + + # Add the disclaimer + final_comment += ( "---\n" "**In case of issue, share mount command, config file and debug-logs to investigate further.**\n\n" "---\n" diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 13dd89ab4d..37af81b181 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -22,7 +22,6 @@ def extract_text_field(text_content: str) -> str: if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') cleaned_text = decoded_text.split("Notes:")[0].strip() - cleaned_text = cleaned_text.split("View this search on")[0].strip() return cleaned_text except re.error as e: print(f"Regular expression error: {e}") From 2da022e0fcda71d4b833fe7d34b1cd6ce48a0c4e Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:20:37 -0700 Subject: [PATCH 14/60] Add error handling --- .github/scripts/ai_summarize.py | 11 ++++++++--- .github/scripts/deepwiki_query.py | 2 -- .github/workflows/aicomment.yml | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 227e9e22e7..e364dd55f6 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -103,13 +103,18 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): if result.returncode == 0: full_text = result.stdout else: - print(f"\n--- Script Error (stderr) ---") - print(result.stderr) + print(f"Error extracting text: {result.stderr}", file=sys.stderr) + sys.exit(1) resp_len = len(full_text) + if resp_len <= 100: + # If the extracted text is too short, we skip commenting on the issue + print("Error: Extracted text is too short to summarize.", file=sys.stderr) + sys.exit(1) # Use the LLM to summarize the text summary = summarize_text_with_llm(full_text) + summary_len = len(summary) final_comment = ( "### AI Generated Response\n\n" @@ -117,7 +122,7 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): f"{summary}\n\n" ) - if resp_len > 65000: + if (resp_len + summary_len) < 65000: final_comment += ( "**Details**\n\n" f"{full_text}\n\n" diff --git a/.github/scripts/deepwiki_query.py b/.github/scripts/deepwiki_query.py index e1fc3c8fa7..8c957c0123 100755 --- a/.github/scripts/deepwiki_query.py +++ b/.github/scripts/deepwiki_query.py @@ -38,8 +38,6 @@ async def ask_deepwiki(self, repo: str, question: str) -> str: if not self.session: raise RuntimeError("Client not connected.") - # print(f"Calling 'ask_question' for repo: {repo}") - # The MCP SDK handles the JSON-RPC call for you result = await self.session.call_tool( "ask_question", {"repoName": repo, "question": question} diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 37c4b97912..aff0930f70 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -43,6 +43,13 @@ jobs: # Run the python script and capture its output to a file python .github/scripts/deepwiki_query.py "${{ env.TARGET_REPO }}" "${{ env.ISSUE_TITLE }}" "${{ env.ISSUE_BODY }}" > deepwiki_full_response.md + # If above script failed then fail the workflow + - name: Check DeepWiki Query Success + if: ${{ failure() }} + run: | + echo "DeepWiki query failed. Exiting workflow." + exit 1 + - name: Print DeepWiki response run: cat deepwiki_full_response.md @@ -51,6 +58,13 @@ jobs: run: | python .github/scripts/ai_summarize.py deepwiki_full_response.md response_comment.md + # If above script failed then fail the workflow + - name: Check Summarization Success + if: ${{ failure() }} + run: | + echo "Summarization failed. Exiting workflow." + exit 1 + - name: Print summarized response run: cat response_comment.md From eb4906956d592a12d6a23e68082f348f7ad8dca5 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:25:01 -0700 Subject: [PATCH 15/60] Updated --- .github/scripts/extract_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 37af81b181..f625272884 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -21,7 +21,7 @@ def extract_text_field(text_content: str) -> str: match = re.search(pattern, text_content, re.DOTALL) if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') - cleaned_text = decoded_text.split("Notes:")[0].strip() + cleaned_text = decoded_text.split("### Notes")[0].strip() return cleaned_text except re.error as e: print(f"Regular expression error: {e}") From 1b9a595be2959156613dec49ff9f4f1fdfec6462 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:29:34 -0700 Subject: [PATCH 16/60] Updated --- .github/scripts/ai_summarize.py | 4 ++-- .github/scripts/extract_text.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index e364dd55f6..108649cb81 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -118,13 +118,13 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): final_comment = ( "### AI Generated Response\n\n" - "**Summary**\n\n" + "####Summary**\n\n" f"{summary}\n\n" ) if (resp_len + summary_len) < 65000: final_comment += ( - "**Details**\n\n" + "####Details**\n\n" f"{full_text}\n\n" ) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index f625272884..52fba5eec3 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -21,7 +21,7 @@ def extract_text_field(text_content: str) -> str: match = re.search(pattern, text_content, re.DOTALL) if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') - cleaned_text = decoded_text.split("### Notes")[0].strip() + cleaned_text = decoded_text.split("## Notes\\n")[0].strip() return cleaned_text except re.error as e: print(f"Regular expression error: {e}") From fb4ec2ff6d458802d82b224ae299164fb09c6497 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:34:32 -0700 Subject: [PATCH 17/60] Updated --- .github/workflows/aicomment.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index aff0930f70..ce29fecad5 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -68,18 +68,6 @@ jobs: - name: Print summarized response run: cat response_comment.md - - name: Read comment body from file - id: get-body - run: | - # Use a delimiter to correctly handle multi-line strings with GITHUB_OUTPUT - content=$(cat response_comment.md) - echo "COMMENT_BODY<> $GITHUB_OUTPUT - echo "$content" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Debug - Print comment body for discussion - run: echo "${{ steps.get-body.outputs.COMMENT_BODY }}" - - name: Post Comment to Issue if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 @@ -88,11 +76,11 @@ jobs: body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} - - name: Post Comment to Discussion - if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - uses: wesleyscholl/create-discussion-comment@v1.0.18 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: ${{ steps.get-body.outputs.COMMENT_BODY }} - discussion-id: ${{ github.event.discussion.id }} - client-mutation-id: "unique-id-1234" \ No newline at end of file + # - name: Post Comment to Discussion + # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + # uses: wesleyscholl/create-discussion-comment@v1.0.18 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # body: ${{ steps.get-body.outputs.COMMENT_BODY }} + # discussion-id: ${{ github.event.discussion.id }} + # client-mutation-id: "unique-id-1234" \ No newline at end of file From b309734306cae4cac1c61ed86c6bafe913c6bd38 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:41:21 -0700 Subject: [PATCH 18/60] Updated --- .github/scripts/ai_summarize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 108649cb81..0cbb6a9316 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -117,14 +117,14 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): summary_len = len(summary) final_comment = ( - "### AI Generated Response\n\n" - "####Summary**\n\n" + "## Summary \n\n" f"{summary}\n\n" ) if (resp_len + summary_len) < 65000: + full_text = full_text.replace("## ", "### ") final_comment += ( - "####Details**\n\n" + "## Details \n\n" f"{full_text}\n\n" ) From d84a6a026aeea17782750e7a7dcff8894d412ec6 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Wed, 17 Sep 2025 23:57:40 -0700 Subject: [PATCH 19/60] Updated --- .github/scripts/extract_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 52fba5eec3..cbca1bdee2 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -21,7 +21,7 @@ def extract_text_field(text_content: str) -> str: match = re.search(pattern, text_content, re.DOTALL) if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') - cleaned_text = decoded_text.split("## Notes\\n")[0].strip() + cleaned_text = decoded_text.split("## Notes\n")[0].strip() return cleaned_text except re.error as e: print(f"Regular expression error: {e}") From d4f82b0f8c2b081a41d5f092cf9e91e22ec1128d Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 00:10:15 -0700 Subject: [PATCH 20/60] Updated --- .github/workflows/aicomment.yml | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index ce29fecad5..0a0fb5e65c 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -12,7 +12,7 @@ jobs: permissions: id-token: write - contents: write + contents: read issues: write discussions: write @@ -76,6 +76,39 @@ jobs: body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read thank you message from file + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + id: read_body + run: | + body=$(> "$GITHUB_OUTPUT + + + - name: Post Thank You Comment + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const discussion_id = ${{ github.event.discussion.id }}; + const body = `${{ steps.read_body.outputs.body }}`; + await github.graphql(` + mutation { + addDiscussionComment(input: { + discussionId: "${discussion_id}", + body: "${body}" + }) { + comment { + id + } + } + } + `); + # - name: Post Comment to Discussion # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' # uses: wesleyscholl/create-discussion-comment@v1.0.18 From c3c1002eabfd6fad144d3d3299a6a274d3a82975 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 00:28:04 -0700 Subject: [PATCH 21/60] Updated --- .github/workflows/aicomment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 0a0fb5e65c..808a8f08c5 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -85,7 +85,7 @@ jobs: body="${body//'%'/'%25'}" body="${body//$'\n'/'%0A'}" body="${body//$'\r'/'%0D'}" - echo "body=$body" >> "$GITHUB_OUTPUT + echo "body=$body" >> "$GITHUB_OUTPUT" - name: Post Thank You Comment From 40b72abd312ca29e157ae66b7ed080b61d1c748b Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 00:51:28 -0700 Subject: [PATCH 22/60] Updated --- .github/workflows/aicomment.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 808a8f08c5..a6aa5cfd65 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -77,18 +77,15 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - - name: Read thank you message from file + - name: Read comment summary from file if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' id: read_body run: | body=$(> "$GITHUB_OUTPUT" - - name: Post Thank You Comment + - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' uses: actions/github-script@v7 with: From 10251aa9509dfdc62ef3990d357aba767edc7ce0 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 02:31:47 -0700 Subject: [PATCH 23/60] Updated --- .github/workflows/aicomment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index a6aa5cfd65..91d364ee0b 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -82,6 +82,9 @@ jobs: id: read_body run: | body=$(> "$GITHUB_OUTPUT" From 0d05c1b29cba0bf0be85aaafddd8960ab4e2f4e2 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 02:40:56 -0700 Subject: [PATCH 24/60] Updated --- .github/workflows/aicomment.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 91d364ee0b..42364f6f10 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -76,7 +76,6 @@ jobs: body-path: 'response_comment.md' token: ${{ secrets.GITHUB_TOKEN }} - - name: Read comment summary from file if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' id: read_body @@ -84,9 +83,12 @@ jobs: body=$(> "$GITHUB_OUTPUT" + - name: Print summary body + if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + run: echo "${{ steps.read_body.outputs.body }}" - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' @@ -94,13 +96,17 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | + const fs = require('fs'); + const path = 'response_comment.md'; + const body = fs.readFileSync(path, 'utf8'); const discussion_id = ${{ github.event.discussion.id }}; - const body = `${{ steps.read_body.outputs.body }}`; + #const body = `${{ steps.read_body.outputs.body }}`; await github.graphql(` mutation { addDiscussionComment(input: { discussionId: "${discussion_id}", - body: "${body}" + body: "${body.replace(/"/g, '\\"')}" + # body: "${body}" }) { comment { id From 76b1278f01075e13d2f27a16f4c47112d0212ce2 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 02:48:57 -0700 Subject: [PATCH 25/60] Updated --- .github/workflows/aicomment.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 42364f6f10..3fad56840b 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -89,7 +89,9 @@ jobs: - name: Print summary body if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' run: echo "${{ steps.read_body.outputs.body }}" - + + # body: "${body}" + # const body = `${{ steps.read_body.outputs.body }}`; - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' uses: actions/github-script@v7 @@ -98,15 +100,14 @@ jobs: script: | const fs = require('fs'); const path = 'response_comment.md'; - const body = fs.readFileSync(path, 'utf8'); + const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\\`'); const discussion_id = ${{ github.event.discussion.id }}; - #const body = `${{ steps.read_body.outputs.body }}`; await github.graphql(` mutation { addDiscussionComment(input: { discussionId: "${discussion_id}", body: "${body.replace(/"/g, '\\"')}" - # body: "${body}" + }) { comment { id From 10ba7e832e302dfe6b035a8b548e8c3757d3f635 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 03:03:49 -0700 Subject: [PATCH 26/60] Updated --- .github/scripts/ai_summarize.py | 31 +++++-------------------- .github/scripts/deepwiki_query.py | 37 ++++++++++++++++++++++++++---- .github/workflows/aicomment.yml | 38 +++++++++++++++---------------- 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 0cbb6a9316..0f2b37819c 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -92,44 +92,25 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): with open(input_file_path, 'r', encoding='utf-8') as f: full_text = f.read() - # Call the extract_text.py script to get the cleaned text - result = subprocess.run( - ["python", ".github/scripts/extract_text.py", input_file_path], - capture_output=True, - text=True - ) - - # The result object contains the output - if result.returncode == 0: - full_text = result.stdout - else: - print(f"Error extracting text: {result.stderr}", file=sys.stderr) - sys.exit(1) - - resp_len = len(full_text) - if resp_len <= 100: - # If the extracted text is too short, we skip commenting on the issue - print("Error: Extracted text is too short to summarize.", file=sys.stderr) - sys.exit(1) - # Use the LLM to summarize the text summary = summarize_text_with_llm(full_text) summary_len = len(summary) + resp_len = len(full_text) + summary_len - final_comment = ( + issue_comment_body = ( "## Summary \n\n" f"{summary}\n\n" ) - if (resp_len + summary_len) < 65000: + if resp_len < 65000: full_text = full_text.replace("## ", "### ") - final_comment += ( + issue_comment_body += ( "## Details \n\n" f"{full_text}\n\n" ) # Add the disclaimer - final_comment += ( + issue_comment_body += ( "---\n" "**In case of issue, share mount command, config file and debug-logs to investigate further.**\n\n" "---\n" @@ -137,4 +118,4 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): ) with open(output_file_path, 'w', encoding='utf-8') as f: - f.write(final_comment) + f.write(issue_comment_body) diff --git a/.github/scripts/deepwiki_query.py b/.github/scripts/deepwiki_query.py index 8c957c0123..6ff149dc78 100755 --- a/.github/scripts/deepwiki_query.py +++ b/.github/scripts/deepwiki_query.py @@ -3,6 +3,7 @@ import sys from contextlib import AsyncExitStack from typing import Optional +import subprocess import httpx from mcp import ClientSession @@ -46,25 +47,51 @@ async def ask_deepwiki(self, repo: str, question: str) -> str: # Join the list of content parts into a single string return result.content -async def main(repo, title, body): +async def main(repo, title, body, output_file): client = MCPClient() try: await client.connect_to_sse_server(server_url=MCP_SSE_URL) question = f"{title}\n\n{body}" response = await client.ask_deepwiki(repo, question) - print(response) + + resp_len = len(response) + if resp_len <= 100: + # If the extracted text is too short, we skip commenting on the issue + print("Error: Extracted text is too short to summarize.", file=sys.stderr) + sys.exit(1) + + with open(output_file, 'w', encoding='utf-8') as f: + f.write(response) + + # Call the extract_text.py script to get the cleaned text + result = subprocess.run( + ["python", ".github/scripts/extract_text.py", output_file], + capture_output=True, + text=True + ) + + # The result object contains the output + if result.returncode == 0: + response = result.stdout + else: + print(f"Error extracting text: {result.stderr}", file=sys.stderr) + sys.exit(1) + + with open(output_file, 'w', encoding='utf-8') as f: + f.write(response) finally: await client.cleanup() if __name__ == "__main__": - if len(sys.argv) < 4: - print("Usage: python deepwiki_query.py ") + if len(sys.argv) < 5: + print("Usage: python deepwiki_query.py ") sys.exit(1) repo_arg = sys.argv[1] issue_title = sys.argv[2] issue_body = sys.argv[3] + output_file = sys.argv[4] - asyncio.run(main(repo_arg, issue_title, issue_body)) + asyncio.run(main(repo_arg, issue_title, issue_body, output_file)) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index 3fad56840b..eb72a37b1e 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -41,7 +41,7 @@ jobs: ISSUE_BODY: $(jq -r '.issue.body // .discussion.body' "${{ github.event_path }}") run: | # Run the python script and capture its output to a file - python .github/scripts/deepwiki_query.py "${{ env.TARGET_REPO }}" "${{ env.ISSUE_TITLE }}" "${{ env.ISSUE_BODY }}" > deepwiki_full_response.md + python .github/scripts/deepwiki_query.py "${{ env.TARGET_REPO }}" "${{ env.ISSUE_TITLE }}" "${{ env.ISSUE_BODY }}" deepwiki_response.md # If above script failed then fail the workflow - name: Check DeepWiki Query Success @@ -51,12 +51,12 @@ jobs: exit 1 - name: Print DeepWiki response - run: cat deepwiki_full_response.md + run: cat deepwiki_response.md - name: Summarize DeepWiki Response with AI id: summarization-step run: | - python .github/scripts/ai_summarize.py deepwiki_full_response.md response_comment.md + python .github/scripts/ai_summarize.py deepwiki_response.md issue_comment.md # If above script failed then fail the workflow - name: Check Summarization Success @@ -66,29 +66,29 @@ jobs: exit 1 - name: Print summarized response - run: cat response_comment.md + run: cat issue_comment.md - name: Post Comment to Issue if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.issue.number }} - body-path: 'response_comment.md' + body-path: 'issue_comment.md' token: ${{ secrets.GITHUB_TOKEN }} - - name: Read comment summary from file - if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - id: read_body - run: | - body=$(> "$GITHUB_OUTPUT" - - - name: Print summary body - if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - run: echo "${{ steps.read_body.outputs.body }}" + # - name: Read comment summary from file + # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + # id: read_body + # run: | + # body=$(> "$GITHUB_OUTPUT" + + # - name: Print summary body + # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' + # run: echo "${{ steps.read_body.outputs.body }}" # body: "${body}" # const body = `${{ steps.read_body.outputs.body }}`; @@ -99,7 +99,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const path = 'response_comment.md'; + const path = 'deepwiki_response.md'; const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\\`'); const discussion_id = ${{ github.event.discussion.id }}; await github.graphql(` From f4e75f93042617f6a479b4461e1b20a3a9a5a1c8 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 03:20:39 -0700 Subject: [PATCH 27/60] Updated --- .github/scripts/ai_summarize.py | 9 +++++++- .github/scripts/deepwiki_query.py | 34 ++++------------------------ .github/scripts/extract_text.py | 9 ++++++-- .github/workflows/aicomment.yml | 37 +++++++++++++++++++++++++++---- 4 files changed, 52 insertions(+), 37 deletions(-) diff --git a/.github/scripts/ai_summarize.py b/.github/scripts/ai_summarize.py index 0f2b37819c..281b9cb0f5 100644 --- a/.github/scripts/ai_summarize.py +++ b/.github/scripts/ai_summarize.py @@ -94,9 +94,16 @@ def refine_summary_from_chunks(text, summarizer, tokenizer): # Use the LLM to summarize the text summary = summarize_text_with_llm(full_text) + if summary == "": + print("Error: Could not generate summary.") + sys.exit(1) + summary_len = len(summary) + if summary_len < 50: + print("Error: Generated summary is too short to be meaningful.") + sys.exit(1) + resp_len = len(full_text) + summary_len - issue_comment_body = ( "## Summary \n\n" f"{summary}\n\n" diff --git a/.github/scripts/deepwiki_query.py b/.github/scripts/deepwiki_query.py index 6ff149dc78..2da8ab0b76 100755 --- a/.github/scripts/deepwiki_query.py +++ b/.github/scripts/deepwiki_query.py @@ -54,44 +54,18 @@ async def main(repo, title, body, output_file): question = f"{title}\n\n{body}" response = await client.ask_deepwiki(repo, question) + print (response) - resp_len = len(response) - if resp_len <= 100: - # If the extracted text is too short, we skip commenting on the issue - print("Error: Extracted text is too short to summarize.", file=sys.stderr) - sys.exit(1) - - with open(output_file, 'w', encoding='utf-8') as f: - f.write(response) - - # Call the extract_text.py script to get the cleaned text - result = subprocess.run( - ["python", ".github/scripts/extract_text.py", output_file], - capture_output=True, - text=True - ) - - # The result object contains the output - if result.returncode == 0: - response = result.stdout - else: - print(f"Error extracting text: {result.stderr}", file=sys.stderr) - sys.exit(1) - - with open(output_file, 'w', encoding='utf-8') as f: - f.write(response) - finally: await client.cleanup() if __name__ == "__main__": - if len(sys.argv) < 5: - print("Usage: python deepwiki_query.py ") + if len(sys.argv) < 4: + print("Usage: python deepwiki_query.py ") sys.exit(1) repo_arg = sys.argv[1] issue_title = sys.argv[2] issue_body = sys.argv[3] - output_file = sys.argv[4] - asyncio.run(main(repo_arg, issue_title, issue_body, output_file)) + asyncio.run(main(repo_arg, issue_title, issue_body)) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index cbca1bdee2..869384250b 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -39,12 +39,17 @@ def extract_text_field(text_content: str) -> str: with open(file_path, 'r', encoding='utf-8') as f: input_text = f.read() + input_len = len(input_text) + if input_len < 100: + print("Input text is too short to process.") + sys.exit(1) + extracted_content = extract_text_field(input_text) - if extracted_content: print(extracted_content) else: - print("Failed to extract the 'text' field.") + print("Error: Could not extract text content.") + sys.exit(1) except FileNotFoundError: print(f"Error: The file '{file_path}' was not found.") diff --git a/.github/workflows/aicomment.yml b/.github/workflows/aicomment.yml index eb72a37b1e..cc933c2804 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/aicomment.yml @@ -25,12 +25,15 @@ jobs: with: python-version: '3.11' + # Install dependencies - name: Install dependencies run: | sudo apt-get install jq python -m pip install --upgrade pip pip install mcp httpx python-dotenv requests openai transformers torch + # --------------------------------------------------------------------------------------------------------------- + # Query DeepWiki using title and body of the issue/discussion - name: Run DeepWiki Query id: deepwiki-step env: @@ -41,7 +44,7 @@ jobs: ISSUE_BODY: $(jq -r '.issue.body // .discussion.body' "${{ github.event_path }}") run: | # Run the python script and capture its output to a file - python .github/scripts/deepwiki_query.py "${{ env.TARGET_REPO }}" "${{ env.ISSUE_TITLE }}" "${{ env.ISSUE_BODY }}" deepwiki_response.md + python .github/scripts/deepwiki_query.py "${{ env.TARGET_REPO }}" "${{ env.ISSUE_TITLE }}" "${{ env.ISSUE_BODY }}" > deepwiki_response.md # If above script failed then fail the workflow - name: Check DeepWiki Query Success @@ -52,11 +55,31 @@ jobs: - name: Print DeepWiki response run: cat deepwiki_response.md + # --------------------------------------------------------------------------------------------------------------- + # Parse DeepWiki response to extract relevant text + - name: Parse DeepWiki Responce and extract text + id: deepwiki-parse-step + run: | + # Run the python script and capture its output to a file + python .github/scripts/extract_text.py deepwiki_response.md > deepwiki_parsed.md + + # If above script failed then fail the workflow + - name: Check DeepWiki Parsing Success + if: ${{ failure() }} + run: | + echo "DeepWiki parsing failed. Exiting workflow." + exit 1 + + - name: Print DeepWiki response + run: cat deepwiki_parsed.md + # --------------------------------------------------------------------------------------------------------------- + + # Summarize DeepWiki response using AI - name: Summarize DeepWiki Response with AI id: summarization-step run: | - python .github/scripts/ai_summarize.py deepwiki_response.md issue_comment.md + python .github/scripts/ai_summarize.py deepwiki_parsed.md issue_comment.md # If above script failed then fail the workflow - name: Check Summarization Success @@ -67,7 +90,9 @@ jobs: - name: Print summarized response run: cat issue_comment.md + # --------------------------------------------------------------------------------------------------------------- + # Post the summary as a comment on the issue - name: Post Comment to Issue if: github.event_name == 'issues' uses: peter-evans/create-or-update-comment@v4 @@ -75,7 +100,9 @@ jobs: issue-number: ${{ github.event.issue.number }} body-path: 'issue_comment.md' token: ${{ secrets.GITHUB_TOKEN }} + # --------------------------------------------------------------------------------------------------------------- + # Post the summary as a comment on discussion # - name: Read comment summary from file # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' # id: read_body @@ -99,7 +126,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const path = 'deepwiki_response.md'; + const path = 'deepwiki_parsed.md'; const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\\`'); const discussion_id = ${{ github.event.discussion.id }}; await github.graphql(` @@ -123,4 +150,6 @@ jobs: # token: ${{ secrets.GITHUB_TOKEN }} # body: ${{ steps.get-body.outputs.COMMENT_BODY }} # discussion-id: ${{ github.event.discussion.id }} - # client-mutation-id: "unique-id-1234" \ No newline at end of file + # client-mutation-id: "unique-id-1234" + + # --------------------------------------------------------------------------------------------------------------- \ No newline at end of file From f60184c064e6a9d3ee6e0ca774438d6d6b528ab8 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 03:25:28 -0700 Subject: [PATCH 28/60] Updated --- .github/scripts/deepwiki_query.py | 2 +- .github/workflows/{aicomment.yml => auto-comment.yml} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{aicomment.yml => auto-comment.yml} (99%) diff --git a/.github/scripts/deepwiki_query.py b/.github/scripts/deepwiki_query.py index 2da8ab0b76..60d2718247 100755 --- a/.github/scripts/deepwiki_query.py +++ b/.github/scripts/deepwiki_query.py @@ -47,7 +47,7 @@ async def ask_deepwiki(self, repo: str, question: str) -> str: # Join the list of content parts into a single string return result.content -async def main(repo, title, body, output_file): +async def main(repo, title, body): client = MCPClient() try: await client.connect_to_sse_server(server_url=MCP_SSE_URL) diff --git a/.github/workflows/aicomment.yml b/.github/workflows/auto-comment.yml similarity index 99% rename from .github/workflows/aicomment.yml rename to .github/workflows/auto-comment.yml index cc933c2804..d5c123d253 100644 --- a/.github/workflows/aicomment.yml +++ b/.github/workflows/auto-comment.yml @@ -1,4 +1,4 @@ -name: Auto Comment Issue with AI +name: Auto Comment Issue OR Discussion on: issues: @@ -7,7 +7,7 @@ on: types: [created] jobs: - ai-comment-on-issue: + auto-comment: runs-on: ubuntu-latest permissions: From 7ebe10c0da69d9adb7175a3aa5da17cd74a50054 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 03:32:31 -0700 Subject: [PATCH 29/60] Updated --- .github/scripts/extract_text.py | 9 +++++++-- .github/workflows/auto-comment.yml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 869384250b..933a47f0cc 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -21,8 +21,13 @@ def extract_text_field(text_content: str) -> str: match = re.search(pattern, text_content, re.DOTALL) if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') - cleaned_text = decoded_text.split("## Notes\n")[0].strip() - return cleaned_text + + # Remove everything after either "## Notes\n" or "Notes:\n" + for delimiter in ["## Notes\n", "Notes:\n"]: + if delimiter in decoded_text: + decoded_text = decoded_text.split(delimiter)[0] + + return decoded_text.strip() except re.error as e: print(f"Regular expression error: {e}") diff --git a/.github/workflows/auto-comment.yml b/.github/workflows/auto-comment.yml index d5c123d253..0d67da2391 100644 --- a/.github/workflows/auto-comment.yml +++ b/.github/workflows/auto-comment.yml @@ -127,7 +127,7 @@ jobs: script: | const fs = require('fs'); const path = 'deepwiki_parsed.md'; - const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\\`'); + const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\`'); const discussion_id = ${{ github.event.discussion.id }}; await github.graphql(` mutation { From 31851c19d219e4311221d6af2cace132fe426301 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 03:45:15 -0700 Subject: [PATCH 30/60] Updated --- .github/workflows/auto-comment.yml | 34 +++++------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/.github/workflows/auto-comment.yml b/.github/workflows/auto-comment.yml index 0d67da2391..84796e5cd2 100644 --- a/.github/workflows/auto-comment.yml +++ b/.github/workflows/auto-comment.yml @@ -103,22 +103,6 @@ jobs: # --------------------------------------------------------------------------------------------------------------- # Post the summary as a comment on discussion - # - name: Read comment summary from file - # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - # id: read_body - # run: | - # body=$(> "$GITHUB_OUTPUT" - - # - name: Print summary body - # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - # run: echo "${{ steps.read_body.outputs.body }}" - - # body: "${body}" - # const body = `${{ steps.read_body.outputs.body }}`; - name: Post Comment to Discussion if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' uses: actions/github-script@v7 @@ -127,14 +111,15 @@ jobs: script: | const fs = require('fs'); const path = 'deepwiki_parsed.md'; - const body = fs.readFileSync(path, 'utf8').replace(/`/g, '\`'); - const discussion_id = ${{ github.event.discussion.id }}; + const body = fs.readFileSync(path, 'utf8') + .replace(/`/g, '\`') // remove backticks + .replace(/"/g, '\"'); // escape double quotes + const discussion_id = context.payload.discussion.node_id; await github.graphql(` mutation { addDiscussionComment(input: { discussionId: "${discussion_id}", - body: "${body.replace(/"/g, '\\"')}" - + body: "${body}" }) { comment { id @@ -143,13 +128,4 @@ jobs: } `); - # - name: Post Comment to Discussion - # if: github.event_name == 'discussion_comment' || github.event_name == 'discussion' - # uses: wesleyscholl/create-discussion-comment@v1.0.18 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # body: ${{ steps.get-body.outputs.COMMENT_BODY }} - # discussion-id: ${{ github.event.discussion.id }} - # client-mutation-id: "unique-id-1234" - # --------------------------------------------------------------------------------------------------------------- \ No newline at end of file From 85fe5ede4584f39e34a331bc1650d61be13063c7 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 18 Sep 2025 04:21:49 -0700 Subject: [PATCH 31/60] Finalizing filter pattern --- .github/scripts/extract_text.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 933a47f0cc..7e0011c822 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -17,17 +17,29 @@ def extract_text_field(text_content: str) -> str: # of the string (', annotations=None, meta=None)]'). This prevents # the regex from stopping at single quotes within the text content. try: - pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" + pattern = "" + if 'text="' in text_content: + pattern = r"text=\"(.*)\"(?=, annotations=None, meta=None\)])" + elif "text='" in text_content: + pattern = r"text='(.*)'(?=, annotations=None, meta=None\)])" + else: + print("Error: The input text does not contain a recognizable text field.") + sys.exit(1) + match = re.search(pattern, text_content, re.DOTALL) if match: decoded_text = match.group(1).encode('utf-8').decode('unicode_escape') - + # Remove everything after either "## Notes\n" or "Notes:\n" for delimiter in ["## Notes\n", "Notes:\n"]: if delimiter in decoded_text: decoded_text = decoded_text.split(delimiter)[0] return decoded_text.strip() + else: + print("No match found for the text field.") + sys.exit(1) + except re.error as e: print(f"Regular expression error: {e}") From 7e76bd10fdae2a68bf586ace544084c831a545ff Mon Sep 17 00:00:00 2001 From: vibhansa Date: Mon, 22 Sep 2025 01:53:46 -0700 Subject: [PATCH 32/60] Updated --- .github/scripts/extract_text.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/extract_text.py b/.github/scripts/extract_text.py index 7e0011c822..24f2c3adba 100644 --- a/.github/scripts/extract_text.py +++ b/.github/scripts/extract_text.py @@ -34,6 +34,10 @@ def extract_text_field(text_content: str) -> str: for delimiter in ["## Notes\n", "Notes:\n"]: if delimiter in decoded_text: decoded_text = decoded_text.split(delimiter)[0] + + + # Replace all double quotes with single quotes + decoded_text = decoded_text.replace('"', "'") return decoded_text.strip() else: From 3aa019ccd4849fa4e9b14a4deb90858f0ee7837c Mon Sep 17 00:00:00 2001 From: vibhansa Date: Mon, 22 Sep 2025 04:15:39 -0700 Subject: [PATCH 33/60] Adding script to retreive all issues and pr and normalize the data --- .github/scripts/issue_normalizer.py | 132 ++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 .github/scripts/issue_normalizer.py diff --git a/.github/scripts/issue_normalizer.py b/.github/scripts/issue_normalizer.py new file mode 100644 index 0000000000..278773f7f3 --- /dev/null +++ b/.github/scripts/issue_normalizer.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +""" +This script fetches issues and pull requests from a specified GitHub repository, +separates them, and writes each item directly to its respective JSON Lines (.jsonl) file. +This approach is more memory-efficient than storing all data in memory first. + +It uses environment variables for both authentication and repository details. + +Prerequisites: +- PyGithub library installed (`pip install PyGithub`) +- The following environment variables must be set: + - GITHUB_TOKEN: A GitHub Personal Access Token with 'repo' scope. + - GITHUB_REPO_OWNER: The owner of the GitHub repository. + - GITHUB_REPO_NAME: The name of the GitHub repository. +""" + +import os +import json +from github import Github, GithubException, Auth + +def get_item_comments(item): + """ + Fetches all comments for a given GitHub issue or pull request + and formats them as a list of dictionaries. + """ + comments_list = [] + try: + for comment in item.get_comments(): + comments_list.append({ + 'author': comment.user.login, + 'comment': comment.body.strip(), + }) + except Exception as e: + print(f"Warning: Could not fetch comments for item #{item.number}. Error: {e}") + + return comments_list + +def main(): + """ + Main function to fetch data and write to JSONL files. + """ + # 1. Get configuration from environment variables + github_token = os.environ.get('GITHUB_TOKEN') + repo_owner = os.environ.get('GITHUB_REPO_OWNER') + repo_name = os.environ.get('GITHUB_REPO_NAME') + + if not all([github_token, repo_owner, repo_name]): + print("Error: One or more required environment variables are not set.") + print("Please set GITHUB_TOKEN, GITHUB_REPO_OWNER, and GITHUB_REPO_NAME and try again.") + return + + issues_file_path = 'issues.jsonl' + prs_file_path = 'prs.jsonl' + issues_count = 0 + prs_count = 0 + + try: + # 2. Authenticate with GitHub using the new Auth class + auth = Auth.Token(github_token) + g = Github(auth=auth) + repo = g.get_repo(f"{repo_owner}/{repo_name}") + print(f"Connected to repository: {repo.full_name}\n") + + # 3. Open files for writing at the beginning + with open(issues_file_path, 'w', encoding='utf-8') as issues_file, \ + open(prs_file_path, 'w', encoding='utf-8') as prs_file: + + # 4. Fetch all items (issues and pull requests) + print("Fetching issues and pull requests...") + all_items = repo.get_issues(state='all') + + for item in all_items: + if item.body == "" or item.body is None: + print(f" Skipping item #{item.number} with empty body...") + continue + + if item.get_comments().totalCount == 0: + print(f" Skipping item #{item.number} with no comments...") + continue + + # Common data points for both issues and PRs + common_data = { + 'title': item.title.strip(), + 'body': item.body.strip() if item.body else "", + 'author': item.user.login, + 'comments': get_item_comments(item) + } + + # Differentiate and write to the correct file + if item.pull_request is None: + # It's a standard issue + print(f" Processing issue #{item.number}...") + data_point = { + 'id': f'issue-{item.number}', + 'type': 'issue', + **common_data + } + issues_file.write(json.dumps(data_point) + '\n') + issues_count += 1 + else: + # It's a pull request, check if it was merged + pr = item.as_pull_request() + if pr.merged: + print(f" Processing merged pull request #{item.number}...") + data_point = { + 'id': f'pr-{item.number}', + 'type': 'pr', + **common_data + } + prs_file.write(json.dumps(data_point) + '\n') + prs_count += 1 + else: + print(f" Skipping unmerged pull request #{item.number}...") + + except GithubException as e: + if e.status == 404: + print(f"\nError: The repository '{repo_owner}/{repo_name}' was not found.") + print("Please double-check the repository name and owner.") + print("If it's a private repository, ensure your GITHUB_TOKEN has the 'repo' scope.") + else: + print(f"Error connecting to GitHub or fetching data: {e.data['message']}") + return + except Exception as e: + print(f"An unexpected error occurred: {e}") + return + + print(f"\nSuccessfully wrote {issues_count} issue records to {issues_file_path}") + print(f"Successfully wrote {prs_count} merged pull request records to {prs_file_path}") + +if __name__ == "__main__": + main() From 09f4aee0a9f8c6bd4cef9ee30e30889c297489b3 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Mon, 22 Sep 2025 04:29:53 -0700 Subject: [PATCH 34/60] Updated --- .github/scripts/issue_normalizer.py | 31 ++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/scripts/issue_normalizer.py b/.github/scripts/issue_normalizer.py index 278773f7f3..1612e82b00 100644 --- a/.github/scripts/issue_normalizer.py +++ b/.github/scripts/issue_normalizer.py @@ -17,6 +17,8 @@ import os import json +import sys +from datetime import datetime, timezone from github import Github, GithubException, Auth def get_item_comments(item): @@ -36,7 +38,7 @@ def get_item_comments(item): return comments_list -def main(): +def main(since_date_str): """ Main function to fetch data and write to JSONL files. """ @@ -50,11 +52,25 @@ def main(): print("Please set GITHUB_TOKEN, GITHUB_REPO_OWNER, and GITHUB_REPO_NAME and try again.") return + since_date = None + if since_date_str: + try: + since_date = datetime.fromisoformat(since_date_str).replace(tzinfo=timezone.utc) + print(f"Fetching data created since: {since_date.isoformat()}") + except ValueError: + print("Warning: Invalid since_date format. Fetching all data instead.") + exit(1) + issues_file_path = 'issues.jsonl' prs_file_path = 'prs.jsonl' issues_count = 0 prs_count = 0 + mode = 'a' + if not since_date: + mode = 'w' + print("Warning: Overwriting existing github_data.jsonl file for full sync.") + try: # 2. Authenticate with GitHub using the new Auth class auth = Auth.Token(github_token) @@ -63,12 +79,15 @@ def main(): print(f"Connected to repository: {repo.full_name}\n") # 3. Open files for writing at the beginning - with open(issues_file_path, 'w', encoding='utf-8') as issues_file, \ - open(prs_file_path, 'w', encoding='utf-8') as prs_file: + with open(issues_file_path, mode, encoding='utf-8') as issues_file, \ + open(prs_file_path, mode, encoding='utf-8') as prs_file: # 4. Fetch all items (issues and pull requests) print("Fetching issues and pull requests...") - all_items = repo.get_issues(state='all') + if since_date: + all_items = repo.get_issues(state='all', since=since_date) + else : + all_items = repo.get_issues(state='all') for item in all_items: if item.body == "" or item.body is None: @@ -84,6 +103,7 @@ def main(): 'title': item.title.strip(), 'body': item.body.strip() if item.body else "", 'author': item.user.login, + 'created_at': item.created_at.isoformat(), 'comments': get_item_comments(item) } @@ -129,4 +149,5 @@ def main(): print(f"Successfully wrote {prs_count} merged pull request records to {prs_file_path}") if __name__ == "__main__": - main() + since = sys.argv[1] if len(sys.argv) > 1 else None + main(since) From 1de76bbef4959ad5e995bbaf532fc238e67f108d Mon Sep 17 00:00:00 2001 From: vibhansa Date: Tue, 23 Sep 2025 02:42:52 -0700 Subject: [PATCH 35/60] Addign workflow to dump list of issues and PR every month --- .github/workflows/issue_dumper.yml | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/issue_dumper.yml diff --git a/.github/workflows/issue_dumper.yml b/.github/workflows/issue_dumper.yml new file mode 100644 index 0000000000..9a7f855300 --- /dev/null +++ b/.github/workflows/issue_dumper.yml @@ -0,0 +1,52 @@ +name: Monthly Issue Dump + +on: + schedule: + - cron: '0 0 1 * *' # Runs at 00:00 on the 1st of every month + workflow_dispatch: # Allows manual triggering + +jobs: + dump-issues: + runs-on: ubuntu-latest + timeout-minutes: 60 # Set timeout to 1 hour + + permissions: + id-token: write + contents: read + issues: write + discussions: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install datetime github requests PyGithub + + - name: Run issue dump script + run: python ./github/scripts/issue_normalizer.py + + - name: Create new branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b issue-dump-$(date +'%Y-%m') + mv *.jsonl ./doc + git add . + git commit -m "Monthly issue dump for $(date +'%B %Y')" + git push origin issue-dump-$(date +'%Y-%m') + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: main + branch: issue-dump-$(date +'%Y-%m') + title: "Monthly Issue Dump - $(date +'%B %Y')" From 64dd3e57ce191b49b82554663ebd55c49bbaf688 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Tue, 23 Sep 2025 02:52:28 -0700 Subject: [PATCH 36/60] Adding current issue and pr dump --- .github/workflows/issue_dumper.yml | 2 +- doc/issues.jsonl | 853 +++++++++++++++++++++++++++++ doc/prs.jsonl | 90 +++ 3 files changed, 944 insertions(+), 1 deletion(-) create mode 100644 doc/issues.jsonl create mode 100644 doc/prs.jsonl diff --git a/.github/workflows/issue_dumper.yml b/.github/workflows/issue_dumper.yml index 9a7f855300..d25993e240 100644 --- a/.github/workflows/issue_dumper.yml +++ b/.github/workflows/issue_dumper.yml @@ -38,7 +38,7 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -b issue-dump-$(date +'%Y-%m') - mv *.jsonl ./doc + mv *.jsonl ./doc/ git add . git commit -m "Monthly issue dump for $(date +'%B %Y')" git push origin issue-dump-$(date +'%Y-%m') diff --git a/doc/issues.jsonl b/doc/issues.jsonl new file mode 100644 index 0000000000..4db0347c38 --- /dev/null +++ b/doc/issues.jsonl @@ -0,0 +1,853 @@ +{"id": "issue-1991", "type": "issue", "title": "Mount All with using advanced options can\u00b4t parse attribute", "body": "when using `blobfuse2 mount all`\n\nreceiving\n```\n\n\n> user : TTY=pts/0 ; PWD=/home/user ; USER=root ; COMMAND=/usr/bin/blobfuse2 mount all /mnt/myblob --config-file='/etc/myconfig.yaml\n> pam_unix(sudo:session): session opened for user root(uid=0) by user(uid=1000)\n> LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\n> LOG_CRIT [mount_all.go (154)]: Starting Blobfuse2 Mount All: 2.5.0\n> LOG_CRIT [mount_all.go (155)]: Logging level set to : LOG_DEBUG\n> LOG_TRACE [azstorage.go (84)]: AzStorage::Configure : azstorage\n> LOG_TRACE [config.go (304)]: ParseAndValidateConfig : Parsing config\n> LOG_ERR [azstorage.go (97)]: AzStorage::Configure : Config validation failed [account name not provided]\n```\n\nWhere as i use according to [advancedConfig.yaml ](https://github.com/Azure/azure-storage-fuse/blob/main/setup/advancedConfig.yaml)\n\nthat is myconfig.yaml\n```\nversion: 2\n\nlogging:\n type: syslog\n level: LOG_WARNING\n\ncomponents:\n - libfuse\n - file_cache\n - azstorage\n\nfile_cache:\n path: /mnt/blobcache\n timeout-sec: 120\n\nprofiles:\n blobmnt1:\n azstorage:\n type: block\n account-name: somesink\n container: blobmnt1\n sas: \n\n libfuse:\n mountpoint: /mnt/myblob/blobmnt1\n allow-other: true\n default-permission: 0770\n\n blobmnt2:\n azstorage:\n type: block\n account-name: somesink\n container: blobmnt2\n sas: \n\n libfuse:\n mountpoint: /mnt/myblob/blobmnt2\n allow-other: true\n default-permission: 0770\n```\n\nwhereas visiting [config.go](https://github.com/Azure/azure-storage-fuse/blob/main/component/azstorage/config.go#L307) i am reading the error, eventhough advancedConfig syntax seem to match\n\nam i missing something? thanks for hints", "author": "EWPStanislavKhodorov", "created_at": "2025-09-17T13:35:30+00:00", "comments": [{"author": "EWPStanislavKhodorov", "comment": "i got the config from an intern, who generated it with gpt5 and used some strange linter...\nremoving the profiles after seeing this myself again did the job"}]} +{"id": "issue-1986", "type": "issue", "title": "File open error for storage mounted with subdirectory and block cache.", "body": "When mounting a storage subdirectory with a block cache, opening of files may fail. Using a file cache or without a subdirectory shows no such problems.\n\n$ sha256sum somefile.bin\nLOG_TRACE [libfuse_handler.go (687)]: Libfuse::libfuse_open : somefile.bin\nLOG_TRACE [block_cache.go (385)]: BlockCache::OpenFile : name=somefile.bin, flags=32768, mode=-rwxrwxrwx\nLOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : somefile.bin\nLOG_DEBUG [attr_cache.go (482)]: AttrCache::GetAttr : somefile.bin served from cache\nLOG_ERR [block_cache.go (389)]: BlockCache::OpenFile : Failed to get attr of somefile.bin [no such file or directory]\nLOG_ERR [libfuse_handler.go (721)]: Libfuse::libfuse_open : Failed to open somefile.bin [no such file or directory]\nsha256sum: somefile.bin: No such file or directory\nInside the directory, some files can be opened and others not, seemingly arbitrarily. The success and failures are consistent across remounts though. The storage has a flat namespace and block blobs.\n\nFrom what I can discern, the failing code path is like this:\n\n[attr_cache.go:GetAttr returns an ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/attr_cache/attr_cache.go#L484).\n[block_cache.go:OpenFile picks up the error and returns the same ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/block_cache/block_cache.go#L390).\n[libfuse_handler.go:libfuse_open picks up the error and fails](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/libfuse/libfuse_handler.go#L723).\nconfig.yaml:\n\nallow-other: true\n\nlogging:\n type: syslog\n level: log_debug\n\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n fuse-trace: true\n\nblock_cache:\n path: /tmp/blobfuse/block_cache\n block-size-mb: 32\n mem-size-mb: 8192\n prefetch: 20\n parallelism: 32\n\nfile_cache:\n path: /tmp/blobfuse/file_cache\n\nattr_cache:\n timeout-sec: 7200\n\nazstorage:\n type: block\nInvoking blobfuse2 with --subdirectory=somefolder then may exhibit this failure. Not passing the subdirectory option or using a file cache seems to work well.\n\nThis was all done in wsl2.", "author": "vibhansa-msft", "created_at": "2025-09-17T07:33:41+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n The problem occurs when mounting a subdirectory with a block cache, and not with a file cache or without a sub directory . The issue appears to stem from the `attr_cache` component failing to retrieve file attributes . This suggests a potential interaction issue between the `subdirectory` option and the 'attr_Cache' component .\n\n---\n**Kindly share mount command, config file and debug-logs for further investigation.**\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}]} +{"id": "issue-1984", "type": "issue", "title": "File open error for storage mounted with subdirectory and block cache.", "body": "When mounting a storage subdirectory with a block cache, opening of files may fail. Using a file cache or without a subdirectory shows no such problems.\n\n$ sha256sum somefile.bin\nLOG_TRACE [libfuse_handler.go (687)]: Libfuse::libfuse_open : somefile.bin\nLOG_TRACE [block_cache.go (385)]: BlockCache::OpenFile : name=somefile.bin, flags=32768, mode=-rwxrwxrwx\nLOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : somefile.bin\nLOG_DEBUG [attr_cache.go (482)]: AttrCache::GetAttr : somefile.bin served from cache\nLOG_ERR [block_cache.go (389)]: BlockCache::OpenFile : Failed to get attr of somefile.bin [no such file or directory]\nLOG_ERR [libfuse_handler.go (721)]: Libfuse::libfuse_open : Failed to open somefile.bin [no such file or directory]\nsha256sum: somefile.bin: No such file or directory\nInside the directory, some files can be opened and others not, seemingly arbitrarily. The success and failures are consistent across remounts though. The storage has a flat namespace and block blobs.\n\nFrom what I can discern, the failing code path is like this:\n\n[attr_cache.go:GetAttr returns an ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/attr_cache/attr_cache.go#L484).\n[block_cache.go:OpenFile picks up the error and returns the same ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/block_cache/block_cache.go#L390).\n[libfuse_handler.go:libfuse_open picks up the error and fails](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/libfuse/libfuse_handler.go#L723).\nconfig.yaml:\n\nallow-other: true\n\nlogging:\n type: syslog\n level: log_debug\n\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n fuse-trace: true\n\nblock_cache:\n path: /tmp/blobfuse/block_cache\n block-size-mb: 32\n mem-size-mb: 8192\n prefetch: 20\n parallelism: 32\n\nfile_cache:\n path: /tmp/blobfuse/file_cache\n\nattr_cache:\n timeout-sec: 7200\n\nazstorage:\n type: block\nInvoking blobfuse2 with --subdirectory=somefolder then may exhibit this failure. Not passing the subdirectory option or using a file cache seems to work well.\n\nThis was all done in wsl2.", "author": "vibhansa-msft", "created_at": "2025-09-17T07:20:10+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n The problem appears to stem from the `attr_cache.go:GetAttr` function returning an 'ENOENT' error . This suggests the attribute cache or storage component is unable to locate the file when a subdirectory is involved and block caching is enabled . The fix in version 2.4.1 for subdirectory path handling supports this hypothesis .\n\n---\n**Kindly share mount command, config file and debug-logs for further investigation.**\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}]} +{"id": "issue-1982", "type": "issue", "title": "Cache not being cleaned up while using high-threshold option", "body": "Which version of blobfuse was used?\n2.0.5, 2.0.6-preview.1 and 2.1.0-preview.2\n\nWhich OS distribution and version are you using?\nUbuntu 22.04\n\nIf relevant, please share your mount command.\nNot exactly the mount command, but my .yaml file used to mount the volume:\n\nallow-other: true #|false \n\n# Logger configuration\nlogging:\n type: base # syslog|silent|base \n level: log_debug # log_off|log_crit|log_err|log_warning|log_info|log_trace|log_debug \n file-path: /var/log/logfile-blobfuse2.log #\n max-file-size-mb: 1024 #\n track-time: true # \ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\nlibfuse:\n disable-writeback-cache: true\n ignore-open-flags: true\n max-fuse-threads: 64\n fuse-trace: false\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n# Disk cache related configuration\nfile_cache:\n # Required\n path: /cachedata/blobfusetmp\n # Optional\n policy: lru # \n timeout-sec: 1209600 # - Try to evict a file only after 14 days\n high-threshold: 5 # <% disk space consumed which triggers eviction. This parameter overrides 'timeout-sec' parameter and cached files will be removed even if they have not expired. Default - 80> - Start eviction with files from here instead of DVR\n low-threshold: 2 # <% disk space consumed which triggers eviction to stop when previously triggered by the high-threshold. Default - 60>\nattr_cache:\n timeout-sec: 7200\nazstorage:\n account-name: # \n account-key: # \n endpoint: # \n container: # \n update-md5: false\n validate-md5: false\n virtual-directory: true\n block-size-mb: 32\n max-concurrency: 8\n disable-compression: true\n sdk-trace: false # \nWhat was the issue encountered?\nIn a similar fashion of the https://github.com/Azure/azure-storage-fuse/issues/1114 the cache are not being cleaned up based on configuration parameters. The documentation states the parameter high-threshold overrides the value set to timeout-sec, but that's not happening.\n\nHave you found a mitigation/solution?\nTried to use a compiled version from main (2.0.6-preview.1) and from 2.1.0-preview.2 with no luck (looks like the changes introduced https://github.com/Azure/azure-storage-fuse/pull/1213 didn't fixed this issue)", "author": "vibhansa-msft", "created_at": "2025-09-17T07:07:42+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n blobfuse2 is not cleaning up cached files based on the `high-threshold` configuration, despite the documentation stating it should override `timeout-sec` This behavior is observed across versions 2.0.5, 2.1.6-preview.1, and 2.2.2 on Ubuntu 22.04 .\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}, {"author": "vibhansa-msft", "comment": "**kindly share config file**"}]} +{"id": "issue-1980", "type": "issue", "title": "direct-io is making too many rest calls", "body": "I want immediiate refresh of my files but using direct-io is making too many REST call.\nhow can i reduce my calls to storage.", "author": "vibhansa-msft", "created_at": "2025-09-17T06:45:58+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n Direct-io bypasses the kernel page cache and Blobfuse2's data and metadata caching . This behavior inherently leads to more REST calls because data is not served from a local cache . To reduce calls to storage while still aiming for file freshness, you should consider adjusting caching mechanisms .\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}]} +{"id": "issue-1979", "type": "issue", "title": "Confused about caching model", "body": "I see there are various kind of cache options available in blobfuse like file-cache, block-cache, attr-cache etc. I am working with very large files which can not fit in my disk, what config shall i use.", "author": "vibhansa-msft", "created_at": "2025-09-17T06:44:34+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n Blobfuse2 offers different caching strategies, and for large files that cannot fit on disk, `block_cache` is the recommended option . This mode performs block-level caching, meaning it downloads and caches parts of the file (blocks) rather than the entire file .\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}]} +{"id": "issue-1978", "type": "issue", "title": "File open error for storage mounted with subdirectory and block cache.", "body": "When mounting a storage subdirectory with a block cache, opening of files may fail. Using a file cache or without a subdirectory shows no such problems.\n\n$ sha256sum somefile.bin\nLOG_TRACE [libfuse_handler.go (687)]: Libfuse::libfuse_open : somefile.bin\nLOG_TRACE [block_cache.go (385)]: BlockCache::OpenFile : name=somefile.bin, flags=32768, mode=-rwxrwxrwx\nLOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : somefile.bin\nLOG_DEBUG [attr_cache.go (482)]: AttrCache::GetAttr : somefile.bin served from cache\nLOG_ERR [block_cache.go (389)]: BlockCache::OpenFile : Failed to get attr of somefile.bin [no such file or directory]\nLOG_ERR [libfuse_handler.go (721)]: Libfuse::libfuse_open : Failed to open somefile.bin [no such file or directory]\nsha256sum: somefile.bin: No such file or directory\nInside the directory, some files can be opened and others not, seemingly arbitrarily. The success and failures are consistent across remounts though. The storage has a flat namespace and block blobs.\n\nFrom what I can discern, the failing code path is like this:\n\n[attr_cache.go:GetAttr returns an ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/attr_cache/attr_cache.go#L484).\n[block_cache.go:OpenFile picks up the error and returns the same ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/block_cache/block_cache.go#L390).\n[libfuse_handler.go:libfuse_open picks up the error and fails](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/libfuse/libfuse_handler.go#L723).\nconfig.yaml:\n\nallow-other: true\n\nlogging:\n type: syslog\n level: log_debug\n\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n fuse-trace: true\n\nblock_cache:\n path: /tmp/blobfuse/block_cache\n block-size-mb: 32\n mem-size-mb: 8192\n prefetch: 20\n parallelism: 32\n\nfile_cache:\n path: /tmp/blobfuse/file_cache\n\nattr_cache:\n timeout-sec: 7200\n\nazstorage:\n type: block\nInvoking blobfuse2 with --subdirectory=somefolder then may exhibit this failure. Not passing the subdirectory option or using a file cache seems to work well.\n\nThis was all done in wsl2.", "author": "vibhansa-msft", "created_at": "2025-09-17T06:40:09+00:00", "comments": [{"author": "github-actions[bot]", "comment": "### AI Generated Response\n\n The issue manifests as a \"No such file or directory\" error during the OpenFile operation . This problem does not occur when using a file cache or when mounting without a subdirectory . The problem likely stems from an incorrect path translation or handling within the `attr_cache` or `azstorage` components .\n\n---\n*Disclaimer: This summary is AI-generated and may not be fully accurate.*"}]} +{"id": "issue-1951", "type": "issue", "title": "BlockCache::OpenFile : Failed to get block list (BlobFuse2)", "body": "**Which version of blobfuse was used?**\nblobfuse2 --version\nblobfuse2 version 2.5.0\n\nrpm -qa | grep -E 'fuse'\nblobfuse2-2.5.0-1.x86_64\nfuse3-libs-3.6.1-4.el7.x86_64\nblobfuse-1.4.2-1.el7_9.x86_64\nfuse-2.9.2-11.el7.x86_64\nfuse-libs-2.9.2-11.el7.x86_64\nfuse3-3.6.1-4.el7.x86_64\n\n\n**Which OS distribution and version are you using?**\n\nSpident\nRed Hat Enterprise Linux Server release 7.9 (Maipo) x86_64\n\nuname -a\nLinux azr-eus2l1530 3.10.0-1160.136.1.el7.x86_64 #1 SMP Tue Jul 8 07:32:00 EDT 2025 x86_64 x86_64 x86_64 GNU/Linux\n\n\n**If relevant, please share your mount command.**\n\n/usr/bin/blobfuse2 mount /oracle/Blob_BAK --config-file=Blobfuse2-BlockCacheConfig2.yaml\n\ncat Blobfuse2-BlockCacheConfig2.yaml\nallow-other: true\nlogging:\n type: base\n level: log_debug\n file-path: /var/log/blobfuse2/blobfuse2_blockcache.log\n max-file-size-mb: 64\n file-count: 2\n track-time: true\nlibfuse:\n fuse-trace: true\n default-permission: 0777\n attribute-expiration-sec: 240\n entry-expiration-sec: 240\n negative-entry-expiration-sec: 120\n ignore-open-flags: true\n read-only: true\nblock_cache:\n block-size-mb: 64\n mem-size-mb: 10240\n prefetch: 12 # Number of blocks to prefetch for sequential reads (e.g., RMAN read-backups)\n parallelism: 128 # Number of parallel upload/download threads operating with disk cache\n path: /mnt/resource/blobfuse2tmp_block\n disk-size-mb: 81920\n disk-timeout-sec: 60\nazstorage:\n type: block\n account-name: xxxx\n account-key: xxxx\n endpoint: xxxxx\n container: xxxxx\n max-concurrency: 24\n update-md5: false\n validate-md5: false\n virtual-directory: true\n max-results-for-list: 2\n disable-compression: false\n telemetry: \"\"\n honour-acl: false\n cpk-enabled: false\n cpk-encryption-key: \"\"\n cpk-encryption-key-sha256: \"\"\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\n**What was the issue encountered?**\n\nThis is the same issue reported previously in https://github.com/Azure/azure-storage-fuse/issues/1536\n\nWed Aug 27 09:17:51 EDT 2025 : blobfuse2[19282] : [/oracle/Blob_BAK] LOG_DEBUG [utils.go (167)]: SDK(Retry) : response 200\nWed Aug 27 09:17:51 EDT 2025 : blobfuse2[19282] : [/oracle/Blob_BAK] LOG_DEBUG [utils.go (167)]: SDK(Retry) : exit due to non-retriable status code\nWed Aug 27 09:17:51 EDT 2025 : blobfuse2[19282] : [/oracle/Blob_BAK] LOG_ERR [block_cache.go (421)]: BlockCache::OpenFile : Failed to get block list of ASD_270536290/ASD_270536290_20250827_94388_1 []\nWed Aug 27 09:17:51 EDT 2025 : blobfuse2[19282] : [/oracle/Blob_BAK] LOG_ERR [libfuse_handler.go (712)]: Libfuse::libfuse_open : Failed to open ASD_270536290/ASD_270536290_20250827_94388_1 [failed to retrieve block list for ASD_270536290/ASD_270536290_20250827_94388_1]\n\nBut the file was created as per the image below (and attached)\n\n\"Image\"\n\n**Have you found a mitigation/solution?**\nNo\n\n**Please share logs if available.**\nAttached", "author": "AdrianAllan0", "created_at": "2025-09-10T17:48:49+00:00", "comments": [{"author": "AdrianAllan0", "comment": "[blobfuse2_blockcache.log](https://github.com/user-attachments/files/22260142/blobfuse2_blockcache.log)"}, {"author": "vibhansa-msft", "comment": "How was this file created ? As per logs when blobfuse tried to retrieve the block-list for this particular blob, from backend we received an empty list."}, {"author": "AdrianAllan0", "comment": "Hi Vikas, thanks for your reply.\nThe file was created by the RMAN archive log backup, I am attaching the output of this execution attached here.\n\nThanks!!!\n\n[rmanBackup_output.txt](https://github.com/user-attachments/files/22274971/rmanBackup_output.txt)"}, {"author": "syeleti-msft", "comment": "Hi @AdrianAllan0 , This is an active bug which we're currently working on, will get resolved in our next release. Thanks"}, {"author": "AdrianAllan0", "comment": "Hi @syeleti-msft, thans for the reply, is there an ETA for this release? Also, is there any possible workaround?"}, {"author": "syeleti-msft", "comment": "next release is scheduled at the month end. you can try use file_cache."}]} +{"id": "issue-1946", "type": "issue", "title": "File cache without kernel cache", "body": "It seems like a common use case for blobfuse2 would be to simply have files downloaded from blob storage when first accessed, cached for a fixed period for future access, and re-downloaded again if accessed after the cache time has expired. During this cache period the file may or may not have been updated externally, but this is acceptable for this hypothetical use case so long as file is not older than the specified cache time. This appears to be straightforward to configure using `--file-cache-timeout` option. \n\nHowever, in my testing it seems like I also need to use `--disable-kernel-cache` or I will keep accessing the old file even after the specified file cache timeout. The default behavior includes kernel caching and this can lead to non-intuitive results if you are not aware of multiple caching layers and interactions. The option to disable kernel caching seems important, yet it doesn't appear on the landing page and is not included in the nice flowcharts on caching.\n\nSomewhat related is the `--direct-io` option and I found some discussion of this related to resolving caching issues. However, this seems like an extreme option because it not only disables kernel caching, but also the blobfuse2 file caching. This seems like a less common use case, but it has more discussion than the option of simply disabling kernel cache.\n\nIs my understanding correct that we must use `--disable-kernel-cache` to ensure blobfuse2 can manage the cache in accordance with the specified `--file-cache-timeout`?\n\nWould it make sense to update the documentation to better explain the `--disable-kernel-cache` option?\n\nBy the way, I am using blobfuse with Azure Batch as configured using Azure Batch SDK (https://learn.microsoft.com/en-us/azure/batch/virtual-file-mount?tabs=windows). I am not sure if this might be causing any atypical behavior, but it does add another layer of abstraction making it harder to configure and debug issues.", "author": "natbprice", "created_at": "2025-09-04T16:17:55+00:00", "comments": [{"author": "vibhansa-msft", "comment": "There are two levels of caching when you are using Blobfuse2 to mount your container.\n- Linux kernel caches both the data and metadata of a file being operated on.\n- Blobfuse level data and metadata caching\n\nKernel level metadata caching can be controlled using timeouts configured while mounting, but kernel data cache can only be turned on or off and not controlled with timeouts. Once kernel caches the data, it will continue serving it from its cache as long as the page cache is valid. Only way to clear it on demand is to wipe out entire kernel cache. \nBlobfuse level cache can be controlled with timeouts (file-cache and attr-cache timeouts).\n\nWhen 'direct-io' is used both kernel and blobfuse level data and metadata cache is disabled. This means you get refreshed contents of files you are using but at a cost of higher number of calls to storage container as every call may results in REST call to container.\n\n'disable-kernel-cache' is another option which allows you to just disable the kernel level data and metadata caches while blobfuse level cache remains and user can configure the respective timeouts for them. This allows you to fine-tune refresh intervals based on your application expectations. If you wish to retrieve the refreshed contents of a file within some given interval, then set file-cache-timeout and attr-cache-timeout to that number (say 5 second for example) and disable the kernel cache using this CLI."}, {"author": "vibhansa-msft", "comment": "[this](https://github.com/Azure/azure-storage-fuse/wiki/Config-File-Best-Practices) wiki page explains this."}]} +{"id": "issue-1944", "type": "issue", "title": "How to control listblobs maxresults argument?", "body": "We are running AKS 1.31 using Blobfuse CSI driver.\nWe have been looking into diagnostics due to increased cost when using Azure Vault Backups and also when disabling cache.\nUsing Log Analytics Workspace we see a lot of ListBlobs calls (not unusual of course), but notice that all calls have \"maxresults=2\" set:\n\n`https://xxxxxxxxx.blob.core.windows.net:443/preprod-sharedstorage?comp=list&delimiter=%2F&include=metadata&maxresults=2&prefix=workarea%2Fhealthcheck%2Fproblems%2FLargeCommit.xml&restype=container`\n\nWe would like to control the maxresults, but we can't figure out how - and if possible at all.\nWhy is it set to 2?\nAny concerns if changing it?", "author": "mortenjoenby", "created_at": "2025-09-04T11:17:29+00:00", "comments": [{"author": "mortenjoenby", "comment": "Seems on 2.5.0 it's set to 2 and does have a configurable setting:\nhttps://github.com/Azure/azure-storage-fuse/blob/blobfuse2-2.5.0-preview.1/component/azstorage/config.go#L188"}, {"author": "vibhansa-msft", "comment": "Hi @mortenjoenby, \"maxresults=2\" is used by Blobfuse when listBlob api is used to retrieve properties of a file. As you can see, in the sample request that you have posted also has file/directory name \"prefix=\". These are special calls and not trying to list all objects in a directory.\nWhen you execute a 'ls' command, you will observe \"maxResults=5000\" in those calls, which is max that Azure storage backend supports. Config that you have pointed is not a general-purpose config and shall not be used by customers. It's meant only to simulate certain scenarios during our testing."}]} +{"id": "issue-1940", "type": "issue", "title": "Mostly empty containers on mount all", "body": "Hi,\n\nI have been trying to mount all of the containers in my storage account on my azure batch nodes. What I notice is that blobfuse reports the mount all operation as successful, but most of the containers are empty(some have data in them, but quite few). The command I perform:\n```\nblobfuse2 mount all testAll/ --config-file=config.yaml --log-level=log_debug --virtual-directory --tmp-path=tempCache/\n```\nwhere the config file points to my storage account and uses MSI for authenticaiton. The MSI has Storage Blob Data Contributor and Reader roles. There is also no output suggesting that there are insufficient permissions for performing the mounting as well. I am also making use of the flag virtual-directory, since hierarchical namespace is disabled on the blob containers. \n\nI also noticed that when i want to unmount all, the command fail to unmount only the ones that does not have any data, the rest are successfully unmounted. \n\nI am able to mount a single container and access its data, but mount all fails to do so. I think there are around 4000 containers that are to be mounted. \n\nI am expecting this to work, as it is recommended by Azure Batch. One thing to note is that the mounting is performed as a startup task, so I am performing this on the node after boot. \n\nPerhaps I am doing something wrong or mounting that many containers is not suitable for this. \n\nCould you aid me in this? \n\nCheers,\nTov", "author": "TovTyvold", "created_at": "2025-08-28T10:19:46+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "@TovTyvold, Can you please confirm over these things - \n1. What's the error unmount all is giving.? If it is resource busy that means there is some operation going over that mount path like listing the container is default one.\n2. Verify MSI role scope => If MSI is scoped only to the storage account level, you\u2019re good. But if it\u2019s container-scoped, some containers might not actually be accessible.\n3. Mounting 4000 container are quite high in numbers. At this scale, you may hit:\n API throttling (Azure Storage account limits).\n Long delays in blob listing.\n\nYou can Try out this disabling list-on-mount (avoids enumerating blobs at mount time):\nAdd in config.yaml:\n\nazstorage:\n block-list-on-mount-sec: 10\n\nPlease do share the config file config.yaml used in your command. and also complete log file with log-level set to log_debug which will be present at ~/.blobfuse2/blobfuse2.log if you haven't changed the log path."}, {"author": "jainakanksha-msft", "comment": "Do you really want to mount all the container at once? because it requires a lot of resource from your machine. \nThink of 4000 blobfuse process running individually and asking for the resources."}, {"author": "TovTyvold", "comment": "To answer if we really want to mount them all at once: The idea was and still is; to prototype accessing files on containers using Blobfuse2. If 4000 containers lead to bad performance, then we obviously do not want to go down that path, but we do not know what to expect since we do not have that much experience with it. I am also looking at mounting a few and looking at its performance.\n\n1) Correct, it says that the device or resource is busy.\n2) The scope is on the storage account, not individual containers, should be OK. \n3) It is able to list all of the blobs pretty fast when listing them in the node($ls -l for example). I do see however that in the azure portal and on the node, when trying to access the directory it lags a lot when trying to list it.\n\nconfig:\nazstorage:\n account-name: mystorageaccount\n mode: msi\n container: blobfuse-placeholder\n resid: /path/to/msi\n\nI do not have the log-file at the moment(i thought log_debug would make the output verbose, so i have not looked at the debug file yet), but I will share after the next attempt. The next attempt will include adding: block-list-on-mount-sec: 10. I will let you know how it goes."}, {"author": "vibhansa-msft", "comment": "If you have 4000 containers, do you need to access all the files across all containers from a single node? or the mount of these containers can be spread across multiple nodes, just in case you have some sort of a cluster. One issue that I see here with such high load is mounting 4000 containers on one machine/VM will create 4000 instances of blobfuse application which might internally than impact your performance as all these instances will compete for network and compute resources."}, {"author": "TovTyvold", "comment": "@vibhansa-msft \nWe can definitely spread the mounting to different pools/nodes and we can also specify which containers to mount, but that will require extra work on our implementation and possibly changes that will cause ripples throughout. The thought was if we could utilize any service to avoid altering our own implementation too much. \nIt sounds like mounting around 4000 containers is bad usage of blobfuse2, which is totally fine. As I said, this was purely an attempt to prototype. \nI might still see how it performs with few containers and estimate whether it is worth it or not."}, {"author": "vibhansa-msft", "comment": "Will close this item here as 4K containers is too much stress on a single system to mount. You can do the required experiments and open a new item if you face any challanges. Feel free to reopen if there are further queries on the mount all."}]} +{"id": "issue-1927", "type": "issue", "title": "When blobfuse2 use --use-adls=true, there have AuthorizationFailure", "body": "# blobfuse2 mount /mnt/batch/tasks/fsmounts/internal-timeseries --config-file=output.yaml --use-adls=true\nError: failed to initialize new pipeline [failed to authenticate azstorage credentials with error [Datalake::TestPipeline : [AuthorizationFailure]]]\n\nHowever the command succeed without the --use-adls=true\n# blobfuse2 mount /mnt/batch/tasks/fsmounts/internal-timeseries --config-file=output.yaml \n\nStorage Account configuration looks fine, blobfuse2 configuration looks fine.\nPlease help to check it.", "author": "Rohan128", "created_at": "2025-08-20T09:08:16+00:00", "comments": [{"author": "Rohan128", "comment": "\"Image\""}, {"author": "Rohan128", "comment": "\"Image\""}, {"author": "souravgupta-msft", "comment": "@Rohan128, kindly redact your account key or other credentials before sharing here. I have removed the config file image from your comment."}, {"author": "souravgupta-msft", "comment": "Is your account HNS enabled?"}, {"author": "Rohan128", "comment": "Yes, it is HNS enabled."}, {"author": "FGS-SWirt", "comment": "I am also encountering the same issue. This has been working for quite a while now with no changes on my end, other than updating to blobfuse2 v2.5.0 when it was released. It worked fine up until very recently, so this seems likely to be something changed on the Azure side of things.\n\nRelevant log entry:\n\n`Aug 27 13:40:11 myhostname blobfuse2[87291]: [/path/to/storage] LOG_ERR [datalake.go (203)]: Datalake::TestPipeline : Failed to validate account with given auth GET https://.dfs.core.windows.net/#012--------------------------------------------------------------------------------#012RESPONSE 403: 403 This request is not authorized to perform this operation.#012ERROR CODE: AuthorizationFailure#012--------------------------------------------------------------------------------#012{#012 \"error\": {#012 \"code\": \"AuthorizationFailure\",#012 \"message\": \"This request is not authorized to perform this operation.\\nRequestId:f7b194ea-001f-0014-3c58-17a891000000\\nTime:2025-08-27T13:40:11.9667837Z\"#012 }#012}#012--------------------------------------------------------------------------------`\n\nSome of the properties on the storage account:\n\n\"Image\""}, {"author": "souravgupta-msft", "comment": "> Yes, it is HNS enabled.\n\n@Rohan128, I saw that you are using `block` as account type in your config. Please update that to `adls` and retry. Also, the adls endpoints are of the format `*.dfs.core.windows.net`, unless you are using private endpoints."}, {"author": "souravgupta-msft", "comment": "@FGS-SWirt, can you please share the config file and mount command you are using. Kindly make sure to redact the account credentials in the config."}, {"author": "FGS-SWirt", "comment": "I was able to fix my issue, at least for now. The only change within the blobfuse2 configuration that I made was using the `dfs.core.windows.net` endpoint instead of the `blob.core.windows.net` endpoint (which had been working just fine for over a year; this change alone did not fix the issue). The primary issue here was with the configuration of the storage account in Azure. It appears some default behavior may have changed in regards to how allow networks/IPs were handled, but I can't prove that.\n\nSome background:\n\n- I have two different VMs in Azure. One mounts a blob with HNS enabled, and one mounts a blob without HNS enabled\n- Both have been working just fine into roughly a week ago, upon which the VM that connects to HNS enabled storage started giving the 403 errors mentioned above.\n\nAfter doing some more digging, I realized that the blob that was still working fine had the Azure VM's VNet in the list of allowed virtual networks (Storage Account -> Security + Networking -> Networking -> click Manage under \"Resource settings: Virtual networks, IP addresses, and exceptions). The HNS-enabled blob, for some reason, did not have that one present, but instead had a different virtual network listed.\n\nOnce I added the VM's VNet (and allowing it to create a service endpoint), then blobfuse2 started working again. The odd thing is that I believe it was working previously without the VM's VNet being in the list of allowed virtual networks. Perhaps something changed on the Azure side and it was inherently trusted because it was an Azure VM, but now it isn't?\n\n@Rohan128 I would try looking at the setting I mentioned above and make sure your VM's public IP address/range is allowed. Here is a screenshot of where I'm talking about if the text wasn't clear:\n\n\"Image\"\n\nAfter clicking Manage, you can see the allowed IPs and networks:\n\n\"Image\""}, {"author": "vibhansa-msft", "comment": "If you are mounting HNS account, kindly use \"type: adls\" as mentioned by Sourav earlier, as certain directory level operations like rename/delete might have issues in wrong mode. If you are not sure about the account type, then leave it to blobfuse which can auto detect it. You shall skip the endpoint config as well, and let blobfuse identify it correctly based on account type. Only place where you need to define endpoint is when you are not using standard endpoints or using private/sovereign clouds."}, {"author": "vibhansa-msft", "comment": "If above information is helpful and you have no further queries on this, kindly close the report here."}, {"author": "vibhansa-msft", "comment": "Closing this item here. If you have any further queries, feel free to reopen or create a new item."}]} +{"id": "issue-1918", "type": "issue", "title": "Blobfuse2 2.5.0 Change to How Re-mount is Handled", "body": "We might be an edge case, but we have hundreds of containers we mount up and that list is constantly changing. We periodically run a mount all to add new containers that have been added to our storage account. Previously this worked fine because only the new containers would be mounted and any existing containers failed (skipped essentially). On upgrade from 2.4.2 to 2.5.0 this functionality changed to an unmount and remount which takes significantly, significantly longer. So much so we had to downgrade to 2.4.2 until we can get something else figured out. \n\nDo you have a suggestion about how to handle this scenario post 2.5.0?\n\nThanks !!!", "author": "michaelschmit", "created_at": "2025-08-11T17:34:58+00:00", "comments": [{"author": "syeleti-msft", "comment": "Hi @michaelschmit, In the container list that you are maintaining, are all the containers valid(i.e., every container is present in the storage acnt) in that list when you are mounting every time? Also may I know how much more time it is taking in the new release? \nAre you also seeing unmount and remount of running mountpoints?"}, {"author": "michaelschmit", "comment": "Again, we aren't maintaining a list today since new containers are being added constantly. Instead, we use the `blobfuse2 mount all`. Generally we are only adding containers, but occasionally a container or two might go away as well.\n\nIn regard to timings, on release 2.4.2 running the mount all command executes in 1m39s. Since most of the containers are already mounted. Depending on how many new containers are added, it can take a few seconds or minutes longer.\n\nOn release 2.5.0, the amount of time to run the mount all command jumps significantly. On the order of 10-30 seconds per container (Again we are mounting ~780 containers today), due to the attempt to unmount and remount I imagine.\n\nIn regards to also seeing unmount and remount of running mount points. It seemed as though the unmount and remount is still failing with the error:\n\n`Failed to mount container [container name] : Error: directory is already mounted`"}, {"author": "michaelschmit", "comment": "Just checking in to see if you need any additional debug information. Thanks !!!"}, {"author": "jainakanksha-msft", "comment": "@michaelschmit, we made an enhancement with our mount all feature.\nNow, whenever it mounts all containers, it first checks if any container are already mounted, unmounts them gracefully, and then proceeds to mount everything. \nThis update is intended to make the product more stable, ensuring that mounting does not fail. \nCould you please provide more details about your use case? This will help us recommend the best way for you to perform this process. \nAlso, why do you need to run mount all so frequently? Keep in mind, the application's initial bootup time might increase slightly."}, {"author": "michaelschmit", "comment": "Why not check if the container is already mounted and do nothing, rather than error? I think the unmount and remount is really only a scalable solution for a few containers which means it is not suitable for mount all.\n\nI think I did a good job describing our use case already, but I will do it again. We have hundreds of containers and that list is constantly changing, and generally growing. These containers are being created dynamically by other code in our solution. When a container gets added, in a short amount of time it needs to be available on the file system. Rather than trying to maintain both a list of directories currently mounted and the new ones that need to be mounted by an outside process, we use the mount all command (as I think mount all was intended).\n\nBeyond that, what I don't understand is why I am still seeing failures even after the change to unmount and remount. The containers still fail with the error:\n\n```\nFailed to mount container [container name] : Error: directory is already mounted\n```\n\nIs that a bug in the logic?\n\nI really think some consideration should be made for solutions with hundred of containers. If it is not a viable solution for hundreds of containers, then it really isn't a scalable solution."}, {"author": "jainakanksha-msft", "comment": "@michaelschmit ,\nI\u2019d like to correct my earlier statement. The error:\n`Failed to mount container [container name] : Error: directory is already mounted`\nis expected when a container is already mounted. In this case, Blobfuse simply skips re-mounting during the next `mount all `execution.\nI verified your scenario with a large and dynamically growing set of containers. \nIn this case, Blobfuse correctly mounts only the new containers, while the already-mounted ones are skipped (with the above message). From my testing:\n\n- Mounting an empty container typically takes <1s.\n- If a container has a large number of files, mount time increases since the kernel performs a directory listing (list call).\n- To mitigate this and avoid mount latency being impacted by the number of files, you can use the block-list-on-mount-sec attribute in your configuration (e.g., set to 10s).\n- Validation of existing container mounts usually takes ~100ms.\n\nSo, the overall performance should scale reasonably well, but it\u2019s worth evaluating at your end based on your specific workload.\n\nIf you still observe that mount all is taking longer than expected, could you please share logs for two consecutive `mount all `operations? We\u2019d be happy to analyze them further to identify any bottlenecks."}, {"author": "michaelschmit", "comment": "I'm seeing a significant increase in the amount of time before a container fails with the:\n`Failed to mount container [container name] : Error: directory is already mounted`\n\nerror message on 2.5.0 vs 2.4.2. Is there a particular log I should look at? Thanks."}, {"author": "vibhansa-msft", "comment": "You can enable debug logs and that can help us understand why it took time. If there are large number of containers being mounted then iterating through them all and identifying current directory is mounted or not, will be time consuming. Also, if there are large number of mounts running already your system will be under stress with low free memory and high cpu usage already. Kindly validate these parts and let us know if you see something else here."}]} +{"id": "issue-1867", "type": "issue", "title": "File not found, even though it exists on the bucket", "body": "We mount a bucket successfully and start accessing files. Occasionally, we see errors like this one:\n\n```\nLOG_WARNING [block_blob.go (549)]: BlockBlob::getAttrUsingList : blob a/b/c.h5 does not exist\n```\n\nIn all such cases, the files actually exist on the bucket. Config:\n\n```\nblobfuse2 mount /mnt/data/gcs-data --config-file=/tmp/blobfuse_config.yaml --foreground --container-name= --wait-for-mount=30s --disable-writeback-cache -o allow_other\n```\n\n/tmp/blobfuse_config.yaml:\n```\nlogging:\n type: syslog\n level: LOG_INFO\nlibfuse:\n attribute-expiration-sec: 60\n entry-expiration-sec: 60\n direct-io: true\nattr_cache:\n timeout-sec: 60\nazstorage:\n type: block\ncomponents:\n- libfuse\n- attr_cache\n- azstorage\n```", "author": "Oblynx", "created_at": "2025-07-10T12:57:49+00:00", "comments": [{"author": "Oblynx", "comment": "The bucket is an Azure blob container."}, {"author": "souravgupta-msft", "comment": "@Oblynx, thanks for reaching out. Was the file created after the mount was done and by some other process, not the running Blobfuse process?"}, {"author": "Oblynx", "comment": "No, the file was pre-existing in this case. In the context of a distributed processing framework, I see this problem pretty frequently when the concurrency level increases. As the concurrency level increases, there are many parallel file accesses across many clients that mount the same bucket. However, each concurrent operation accesses a _different_ file. The same file is not accessed by two different operations, and it is never written, only read."}, {"author": "souravgupta-msft", "comment": "Can you please enable debug logs in your config. If `direct-io` mode is used, remove attr_cache from the config and retry."}, {"author": "Oblynx", "comment": "I tried a few things:\n- remove `direct-io`: I no longer see the issue, but performance is lower and memory use much higher (caching is unnecessary in this use case)\n- remove `attr_cache`: I no longer see this issue, but memory use is much higher\n\nMaybe in both cases I haven't allowed enough memory to push the computation long enough to actually encounter the issue...\n@souravgupta-msft what is the relationship between the two settings? Is it not supported to have both? If this is by design, then why does it work to a significant degree, crashing only for some files?"}, {"author": "Oblynx", "comment": "My last comment was premature, checking better."}, {"author": "Oblynx", "comment": "I tried a few things:\n\n- remove `direct-io`: I no longer see the issue, but performance is lower and memory use much higher (caching is unnecessary in this use case). Maybe I haven't allowed enough memory to push the computation long enough to actually encounter the issue...\n- remove `attr_cache`: I still see the issue, and memory use is much higher (4->10GB tested so far).\n ```\n logging:\n type: syslog\n level: LOG_DEBUG\n libfuse:\n attribute-expiration-sec: 60\n entry-expiration-sec: 60\n direct-io: true\n azstorage:\n type: block\n components:\n - libfuse\n - azstorage\n ```\n \n\n@souravgupta-msft what is the relationship between the two settings? Is it not supported to have both? If this is by design, then why does it work to a significant degree, crashing only for some files?"}, {"author": "Oblynx", "comment": "Some debug logs with `attr_cache` disabled:\n```\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_DEBUG [utils.go (162)]: SDK(Retry) : response 200\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_DEBUG [utils.go (162)]: SDK(Retry) : exit due to non-retriable status code\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_TRACE [block_blob.go (542)]: BlockBlob::getAttrUsingList : So far retrieved 0 objects in 1 iterations\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_WARNING [block_blob.go (549)]: BlockBlob::getAttrUsingList : blob .sanctuary/374a9846-d9ba-4f57-adb5-1bdcb7c760e6/sample01.volume_cfd.verts.bin does not exist\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_TRACE [libfuse_handler.go (641)]: Libfuse::libfuse_create : .sanctuary/374a9846-d9ba-4f57-adb5-1bdcb7c760e6/sample01.volume_cfd.verts.bin\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_TRACE [azstorage.go (363)]: AzStorage::CreateFile : .sanctuary/374a9846-d9ba-4f57-adb5-1bdcb7c760e6/sample01.volume_cfd.verts.bin\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_TRACE [block_blob.go (275)]: BlockBlob::CreateFile : name .sanctuary/374a9846-d9ba-4f57-adb5-1bdcb7c760e6/sample01.volume_cfd.verts.bin\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_TRACE [block_blob.go (1126)]: BlockBlob::WriteFromBuffer : name .sanctuary/374a9846-d9ba-4f57-adb5-1bdcb7c760e6/sample01.volume_cfd.verts.bin\nworker-c89pk blobfuse2[24]: [/mnt/data/gcs-data] LOG_DEBUG [utils.go (162)]: SDK(Retry) : =====> Try=1 for PUT https:///.sanctuary%2F374a9846-d9ba-4f57-adb5-1bdcb7c760e6%2Fsample01.volume_cfd.verts.bin\n```"}, {"author": "vibhansa-msft", "comment": "Is the issue here only about observing the log where it says \"file does not exists\" or does it make any failure on the application end. When an application tries to create a new file, linux kernel will try to get the attributes of a file first to check if it exists or not. As part of this call Blobfuse will do a check on the storage and if storage backend returns back 404 or no attributes of the file in listBlob api then a log will be emitted saying \"file not found\". Immediatly after this kernel will create the file"}, {"author": "Oblynx", "comment": "Unfortunately it is the application that sees this error, and that's why I even looked at the fuse logs. The application (python) gets an OSError that the file doesn't exist. These files where the error is being observed were written to blob storage days ago, and are only being **read** by the application."}, {"author": "vibhansa-msft", "comment": "can you enable debug logs and re-run your workflow where application receives this error and then share the log file with us."}, {"author": "Oblynx", "comment": "Here you go. Is there any other info I can provide?\n\n[fuse-gpu-logs.txt](https://github.com/user-attachments/files/21232812/fuse-gpu-logs.txt)"}, {"author": "vibhansa-msft", "comment": "Thanks for sharing the log file, I do see few things which are not tuned correctly. In your config file you are not using any file-cache or block-cache model which means every write is streamed directly to storage which is very costly. For every write operation I do see a list, read and write request going to storage container.\n\nIn your config file, have something like below and configure a temp cache path. \n```\ncomponents:\n- libfuse\n- file_cache\n- attr_cache\n- azstorage\n```\n\nLets fix this and then come back to the file not found issue. After fixing this you may not hit the isuse."}, {"author": "Oblynx", "comment": "Maybe this is not the correct solution, but I removed the caches on purpose. In my application, during the lifetime of the container where blobfuse is running, a single map operation is going to take place data-parallel over a large dataset.\nThis means that every input file is going to be read exactly once and every output file written exactly once => no cache hits.\n\nIn the logs I needed to obfuscate the file names for IP protection and I replaced the names of many different files with the same identifier.\n\nAdditionally, I aim for high performance, eliding any unnecessary copies (especially to disk).\nI previously played with file cache and noticed lower performance, which makes sense given the extra disk write. I also played with an in-memory block cache, which didn't show any performance impact (neither + nor -, since the cache is never hit and memory writes are very fast).\n\nBtw, I've run synthetic benchmarks with `fio` in my environment, before understanding much about the config. We could reproduce those with various different config settings for a more scientific assay...\n\n## Experiments with block-cache\n\n#### Highly parallel read-only workload (read once always)\n\n```\nlogging:\n type: syslog\n level: LOG_INFO\nlibfuse:\n attribute-expiration-sec: 60\n entry-expiration-sec: 60\n direct-io: true\nattr_cache:\n timeout-sec: 60\nazstorage:\n type: block\ncomponents:\n- libfuse\n- block_cache\n- attr_cache\n- azstorage\nblock_cache:\n block-size-mb: 16\n mem-size-mb: 1478\n prefetch: 12\n parallelism: 288\n disk-size-mb: 0\n```\n\nGrafana plot of memory working set:\n\n\"Image\"\n\nUnfortunately, there were some stability issues with this in-memory block cache. I haven't documented them in detail, but open to repeating some experiments. It could be that I ran into the same issue I report here.\n\n## Summary\n\n- In this use case, I believe that caching is useless\n- Encountered some stability issues with the block-cache, which unfortunately I didn't document\n- Facing this bug with no cache\n\n\nA key reason to upgrade from blobfuse1, our current solution, would be predictable and low memory use. I evaluate this in my tests.\n\nLet me know what, if any, experiment you would like from me given this feedback."}, {"author": "vibhansa-msft", "comment": "Thanks for sharing details and context. Not having any cache in pipeline may create more issues than solving them. You can use file-cache with cache-timeout set to 0 so that everytime you close a file, its evicted from local cache and there is no cache hit for that anyway. If the file size that you read or write is large (few GBs) then better to stick to block-cache. Advantage that you get with block-cache is we do not wait for entire file to be written before we upload, rather we upload in parallel to write. Let blobfuse autotune the block-cache config for you by removing entire block-cache secton from config file and then try once."}, {"author": "vibhansa-msft", "comment": "Also you shall not use direct-io as that results in reduced caching on blobfuse and more rest calls to storage."}, {"author": "vibhansa-msft", "comment": "Were you able to validate by putting file-cache or block-cache in the pipeline ?"}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update. Feel free to reopen or post your queries here."}, {"author": "Oblynx", "comment": "Hi, it's still on my backlog and I will return to this. Holiday season in between ;)\n\n@vibhansa-msft I don't have the permissions to re-open"}]} +{"id": "issue-1860", "type": "issue", "title": "Blobfuse2 endpoint (blobfuse2.z13.web.core.windows.net) not documented \u2013 Observed when deploying AKS PV with Azure Firewall", "body": "When creating a Persistent Volume using Blobfuse on AKS, it establishes HTTPS communication with `blobfuse2.z13.web.core.windows.net`. This behavior was confirmed during testing with Azure Firewall. However, this endpoint is not mentioned in the official documentation. Please confirm if this URL is valid and update the documentation accordingly.\n\n```yaml\n# StorageClass\nallowVolumeExpansion: true\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n labels:\n addonmanager.kubernetes.io/mode: EnsureExists\n kubernetes.io/cluster-service: \"true\"\n name: azureblob-fuse-premium\nmountOptions:\n- -o allow_other\n- --file-cache-timeout-in-seconds=120\n- --use-attr-cache=true\n- --cancel-list-on-mount-seconds=10\n- -o attr_timeout=120\n- -o entry_timeout=120\n- -o negative_timeout=120\n- --log-level=LOG_WARNING\n- --cache-size-mb=1000\nparameters:\n skuName: Premium_LRS\nprovisioner: blob.csi.azure.com\nreclaimPolicy: Delete\nvolumeBindingMode: Immediate\n```\n\n```yaml\n# PersistentVolume\napiVersion: v1\nkind: PersistentVolume\nmetadata:\n annotations:\n pv.kubernetes.io/provisioned-by: blob.csi.azure.com\n name: pv-blob\nspec:\n accessModes:\n - ReadWriteOnce\n capacity:\n storage: 128Gi\n csi:\n driver: blob.csi.azure.com\n nodeStageSecretRef:\n name: secrets\n namespace: dev\n volumeAttributes:\n containerName: \n resourceGroup: \n storageAccount: \n volumeHandle: pv-blob-volume-id\n mountOptions:\n - -o allow_other\n - --file-cache-timeout-in-seconds=120\n - --use-attr-cache=true\n - --cancel-list-on-mount-seconds=10\n - -o attr_timeout=120\n - -o entry_timeout=120\n - -o negative_timeout=120\n - --log-level=LOG_WARNING\n - --cache-size-mb=1000\n persistentVolumeReclaimPolicy: Retain\n storageClassName: azureblob-fuse-premium\n volumeMode: Filesystem\n```\n\n```yaml\n# PersistentVolumeClaim\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: pv-blob-pvc\n namespace: dev\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 128Gi\n storageClassName: azureblob-fuse-premium\n volumeMode: Filesystem\n volumeName: pv-blob\n```\n\nReferences\n1) https://learn.microsoft.com/en-us/azure/aks/azure-csi-blob-storage-provision?tabs=mount-nfs%2Csecret&source=docs\n2) https://learn.microsoft.com/en-us/azure/aks/outbound-rules-control-egress?source=docs", "author": "Lddeiva", "created_at": "2025-07-09T03:50:03+00:00", "comments": [{"author": "vibhansa-msft", "comment": "This endpoint is used just to validate whether you are using the latest version or blobfuse or not. This is checked only once when you mount, and connection can be disabled by disabling the version check. Even if the connection does not go through, it does not create any issue in your mount, neither does it complain about anything. This is a silent background check for version. Do you see any issue with this ?"}, {"author": "Lddeiva", "comment": "> This endpoint is used just to validate whether you are using the latest version or blobfuse or not. This is checked only once when you mount, and connection can be disabled by disabling the version check. Even if the connection does not go through, it does not create any issue in your mount, neither does it complain about anything. This is a silent background check for version. Do you see any issue with this ?\n\n@vibhansa-msft , There are no issues with the behavior itself, but it would be helpful to have this documented to provide clarity, address customer concerns, and facilitate coordination with security teams."}, {"author": "vibhansa-msft", "comment": "@tjvishnu Blobfuse checks for latest version on the mentioned endpoint. We might need to update our documentation regarding the same."}]} +{"id": "issue-1843", "type": "issue", "title": "Blobfuse cache files are not released properly, causing blobfuse mounts to fail on system reboot.", "body": "Blobfuse cache files are not released properly, causing blobfuse mounts to fail on system reboot.\n\nWe open ticket with MS for this issue and their ask to open ticket with Blobfuse git repo. The DFS cache and temp empty files are not getting cleanup on system reboot and it fails to mount blobfuse file systems on system reboot.\n\nAttached config file and other details and errors.", "author": "naisu07-us", "created_at": "2025-06-24T16:20:25+00:00", "comments": [{"author": "souravgupta-msft", "comment": "Hi @naisu07-us, thanks for reaching out. Please add `cleanup-on-start: true` in the [file-cache section](https://github.com/Azure/azure-storage-fuse/blob/main/setup/advancedConfig.yaml#L102) of the config and retry your operation."}, {"author": "naisu07-us", "comment": "We used that option with config file, attached both config and file lists with error log while starting blobfuse2 mounts.\n\n[blobfuse2_config_example.txt](https://github.com/user-attachments/files/20887632/blobfuse2_config_example.txt)\n[blobfuse2-failed-to-mount-cache-file-issue.txt](https://github.com/user-attachments/files/20887631/blobfuse2-failed-to-mount-cache-file-issue.txt)"}, {"author": "naisu07-us", "comment": "Any suggestion on this issues ?"}, {"author": "souravgupta-msft", "comment": "Can you also please share your mount command."}, {"author": "souravgupta-msft", "comment": "Also please update your config file like this, https://github.com/Azure/azure-storage-fuse/blob/main/sampleFileCacheConfig.yaml\nMake sure to add these fields in the config file under file-cache section.\n`cleanup-on-start: true` and `allow-non-empty-temp: true`"}, {"author": "naisu07-us", "comment": "Here is start service to mount blobfuse2 container mounts.\n\n[Unit]\nDescription=\"Blobfuse mount for /myblobhome/blobfusetestact/mycontainer-one\"\nAfter=remote-fs.target\nRequires=remote-fs.target\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nUser=b80adm\nGroup=sapsys\nTimeoutStartSec=120sec\nTimeoutStopSec=360sec\nType=simple\nNotifyAccess=all\nKillMode=process\nExecStart=/usr/bin/blobfuse2 mount /myblobhome/blobfusetestact/mycontainer-one --config-file=/etc/blobfuse/storage-account/mycontainer-one_config.yaml -o attr_timeout=240 -o entry_timeout=240 -o negative_timeout=120\nExecStop=/usr/bin/blobfuse2 unmount /myblobhome/blobfusetestact/mycontainer-one\nRemainAfterExit=yes\n\n[Install]\nWantedBy=multi-user.target"}, {"author": "vibhansa-msft", "comment": "You might want to add those additional parameters suggested by Sourav in your ExeStart step. If you want a reference service file you can look at [this](https://github.com/Azure/azure-storage-fuse/blob/main/setup/blobfuse2.service)"}, {"author": "vibhansa-msft", "comment": "Closing this as there are no updates. Feel free to reopen if the issue persists."}]} +{"id": "issue-1839", "type": "issue", "title": "Blobfuse files do not refresh/update after mounting in a container with PV", "body": "Using Blobfuse 2, we have files syncing to the Blockblob storage container from source machine and the same container is mounted on a cluster via PV. \n\nWhen an update to a data file occurs the change gets reflected instantly on the blob container, however the same does not reflect on the pod mount point. \n\nWhat would be the right set of option to be able to see the file update on the pod/cluster without direct_io. \n\nPlease note we tried using -o direct_io, but that causes performance degradation outside the acceptable range for the app. \n\nWith the below configuration the file never updates on the cluster. \n\n**PV yaml**\n\napiVersion: v1\nkind: PersistentVolume\nmetadata:\n annotations:\n pv.kubernetes.io/provisioned-by: blob.csi.azure.com\n name: {{ .Values.common.pv.fileshare.rd.name }}\nspec:\n capacity:\n storage: 1500Gi\n accessModes:\n - ReadWriteMany\n persistentVolumeReclaimPolicy: Retain\n storageClassName: azureblob-fuse-premium\n mountOptions:\n - -o allow_other\n - --file-cache-timeout-in-seconds=15\n - --attr-cache-timeout=15\n \n csi:\n driver: blob.csi.azure.com \n volumeHandle: {{ .Values.common.volumeHandle }}-rdata-zrs\n volumeAttributes:\n containerName: rdata\n nodeStageSecretRef:\n name: {{ .Values.common.group }}-blob-secret\n namespace: {{ .Values.common.namespace }}\n\nPlease let me know if I can share any other details", "author": "nilay9011", "created_at": "2025-06-18T21:11:57+00:00", "comments": [{"author": "vibhansa-msft", "comment": "As per your config \"--file-cache-timeout-in-seconds=15\", blobfuse will keep the file locally cached for 15 seconds. Post this if you try to access the file it shall download a new version from storage. That being said, kernel cache is by default enables and as long as kernel has the contents cached read requests will be served from it. This means calls will not even reach blobfuse.\nIf you wish instant refresh of the contents then you need to disable kernel cache. To do so you need to use \"-o direct_io\" flag in your mount option. Be aware that once you disable both kernel and blobfuse caching, now all calls will have to hit storage to fetch the latest contents and this means increased number of REST calls to storage resulting in high billing."}]} +{"id": "issue-1835", "type": "issue", "title": "If blobfuse is mounted with preload option then file system shall be in read-only mode", "body": "If blobfuse is mounted with --preload option then file system shall be mounted in read-only mode but it still allow me to write/create file.", "author": "vibhansa-msft", "created_at": "2025-06-14T07:36:06+00:00", "comments": [{"author": "vibhansa-msft", "comment": "This was due to error in script where return value of dd command was not checked properly."}]} +{"id": "issue-1821", "type": "issue", "title": "set disable-version-check as true by default", "body": "`disable-version-check` is now true by default, which would produce the version check logs in every mount, can we turn if off by default?\n\nhttps://github.com/Azure/azure-storage-fuse/blob/5e0ecbfd0e63a70dca732601790538e455816b41/cmd/root.go#L292", "author": "andyzhangx", "created_at": "2025-06-05T09:47:56+00:00", "comments": [{"author": "vibhansa-msft", "comment": "For AKS cases it may not add any value but customer mounting from shell is notified about their version being older and an update is available, hence we do not disable this check by default. I guess AKS sending the flag explicitly is the correct thing here. Do you see any challenge with this? That other error about invalid flag, I do not see that in my local setup, and I have not heard any other customer also complaining about this so far."}, {"author": "andyzhangx", "comment": "ok, I will disable version check in blob csi driver"}]} +{"id": "issue-1816", "type": "issue", "title": "Files transfer issue \"An unexpected network error occurred\"", "body": "### Which version of blobfuse was used?\nblobfuse2 version 2.0.0-preview.3\n\n### Which OS distribution and version are you using?\nDebian GNU/Linux 11 (bullseye)\n\n### My fuse_connection.yaml file\n![Image](https://github.com/user-attachments/assets/7f9d4aba-f2a2-4614-b96b-50567a031c24)\n\n### What was the issue encountered?\nWe have identified an issue where file transfer to Azure storage is failing with the error \"An unexpected network error occurred.\"\nIn some cases, this problem may occur several times in a row. For example, our automation has restart attempts with a timeout. Now, the file transfer has failed three times in a row. \n\nWe have provided logs where you can find more detailed information under the file name mkpdev-2180-20250527-* (the file names differ because each attempt creates a file with a different name).\nHere is the approximate time of the fall (in timezone GMT+3):\n1. 16:17:47\u00a0 2. 16:34:21\u00a0 3. 16:46:36\u00a0\n\n**We would like to ask if you can tell us what exactly this problem might be related to and how we can avoid it for sure.**\n\n### Blolfuse2 Logs\n[blobfuse2.zip](https://github.com/user-attachments/files/20575425/blobfuse2.zip)", "author": "olegshv", "created_at": "2025-06-03T14:41:12+00:00", "comments": [{"author": "vibhansa-msft", "comment": "Only error I see in logs is some files fail to flush to storage, as they were not found in local storage. Reason for not finding them in local storage was they were renamed before closing the handle. I see lot of .backup files being created, renamed, deleted etc in quick succession. What kind of tool they are using ?"}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update for 2 weeks. Feel free to reopen or post your comments here if the issue persists."}]} +{"id": "issue-1810", "type": "issue", "title": "Windows support", "body": "Blobfuse works only on linux platform. I tried compiling the code on windows platform but it does not even compile.\nWe need winfsp based fuse implementation in blobfuse code base so that it can be used on windows platform as well.\nThere are some infra components that will need change like syslog is not supported on windows and we might need to integrate windows event logging for that, stats of file-system in windows returns back different data types. Also winfsp integration by searching the dll and loading it dynamically will be needed.", "author": "vibhansa-msft", "created_at": "2025-06-02T10:33:05+00:00", "comments": [{"author": "jfantinhardesty", "comment": "We have a fork of blobfuse2 called [cloudfuse](https://github.com/Seagate/cloudfuse) and have added Windows support. Perhaps we could help assist on the Windows support. We did it by changing the libfuse component to use the [cgofuse](https://github.com/winfsp/cgofuse) library which supports fuse2, fuse3, and winfsp on Windows. It abstracts away some of that logic to make it a bit easier to work with. It also removes much of the need to have two fuse files. There additionally were some other changes needed to the code to fix issues with paths between Linux and Windows as well as additional race conditions that appear in the code on Windows but do not on Linux."}, {"author": "vibhansa-msft", "comment": "@jfantinhardesty we had cgofuse based implementation earlier but we preferred our own implementation over it, so that in future maintenance is easy and we do not have to rely on other third-party tools for fixing critical customer issues. If you wish to contribute to main repo, maybe you can fire a PR in this repo where for windows it relies on cgofuse while Linux implementation remains the same."}, {"author": "jfantinhardesty", "comment": "@vibhansa-msft sure I can try to do the cgofuse conversion for the libfuse component, but there are many aspects to blobfuse2 that have to change for full windows support. In particular, block_cache and xload use Linux specific features, so those will not work right away (though we have gotten block cache support added for Windows). Also, the health monitor and stats manager use Linux features so those must change as well. There are also the path differences between Windows and Linux and specific aspects to WinFSP that add additional complexity. I have done a quick first pass at implementation here, but there is still more that would need to be done to make it work on Windows. Here is the branch that I have started with some initial support https://github.com/jfantinhardesty/azure-storage-fuse/tree/windows-support"}, {"author": "vibhansa-msft", "comment": "Yes I do understand its not a small project and lot of platform specific code needs fixing to bring it up fully. That was one of the reason to assign this to copilot instead of manually making the changes. If it can do 70% of the job then its easier for us to fix the rest :)"}, {"author": "jfantinhardesty", "comment": "@vibhansa-msft Very fair, it is a big project. I did put a bit more work into that branch and it now does at least compile on Windows. So it could be at the point of a good place to start for the Windows support."}]} +{"id": "issue-1790", "type": "issue", "title": "blobfuse2: error while loading shared libraries: libfuse3.so.3", "body": "Got the error on debian.\n`$ sudo dpkg -i Downloads/blobfuse2_2.4.2_amd64.deb`\n\n```\n$ blobfuse2 --v\nblobfuse2: error while loading shared libraries: libfuse3.so.3: cannot open shared object file: No such file or directory\n```\n\nSimilar reports:\nRedhat - https://github.com/Azure/azure-storage-fuse/issues/1039\n\n```\n$ fusermount3 --v\nfusermount3 version: 3.17.2\n```\n\n$ find /usr/lib -name libfuse3*\n/usr/lib/x86_64-linux-gnu/libfuse3.so.3.17.2\n/usr/lib/x86_64-linux-gnu/libfuse3.so.4\n/usr/lib/x86_64-linux-gnu/libfuse3.a\n/usr/lib/x86_64-linux-gnu/libfuse3.so\n\n\nIn summary I think the cause is that latest libfuse3 has lib libfuse3.so.4", "author": "diepes", "created_at": "2025-05-25T01:22:43+00:00", "comments": [{"author": "diepes", "comment": "Work around - follow the README.md and build it, was quite straight forward.\n\n\n```\n sudo apt install fuse3 libfuse3-dev gcc\n # go build -o blobfuse2\n\n~/github$ git clone git@github.com:Azure/azure-storage-fuse.git\n\n~/github/azure-storage-fuse$ go build -o blobfuse2\n~/github/azure-storage-fuse$ ./blobfuse2 -v\nblobfuse2 version 2.5.0~preview.1\n```"}]} +{"id": "issue-1765", "type": "issue", "title": "Support cleanup-on-start or empty-dir-check as BlobFuse2 CLI parameters for block_cache mode", "body": "Context:\n\nWe are using Azure Batch with Azure Linux images to mount Azure Blob Storage using BlobFuse2, following the [official guidance](https://learn.microsoft.com/en-us/azure/batch/virtual-file-mount?tabs=linux).\n\nIn Azure Batch, the only way to configure BlobFuse2 is through the [AzureBlobFileSystemConfiguration.blobfuseOptions field](https://learn.microsoft.com/en-us/azure/templates/microsoft.batch/batchaccounts/pools?pivots=deployment-language-bicep#azureblobfilesystemconfiguration), which passes CLI parameters directly to the BlobFuse2 mount command.\n\nProblem:\n\nWhen our Azure Batch VMs are rebooted (due to OS upgrades or manual operations), BlobFuse2 does not shut down gracefully. Upon restart, it fails with the following error:\n\n`[config error in block_cache \"[temp directory not empty]\"]`\n\nThis happens because the block cache directory is not empty, and BlobFuse2 currently performs a strict check on this.\n\nInvestigation:\n\nThe block_cache component has a cleanup-on-start config option in the\nhttps://github.com/Azure/azure-storage-fuse/blob/4ddfcd4b776650ae5172663c04db2a0fb791cbd6/component/block_cache/block_cache.go#L105\nHowever, this option is not exposed via CLI flags and cannot be passed through blobfuseOptions in Azure Batch.\n\nThe file_cache component supports the --empty-dir-check CLI flag:\nhttps://github.com/Azure/azure-storage-fuse/blob/4ddfcd4b776650ae5172663c04db2a0fb791cbd6/component/file_cache/file_cache.go#L1641\nIt also supports the cleanup-on-start config option:\nhttps://github.com/Azure/azure-storage-fuse/blob/4ddfcd4b776650ae5172663c04db2a0fb791cbd6/component/file_cache/file_cache.go#L105\nThe mount.go logic also references the cleanup-on-start config option for file_cache mode \nhttps://github.com/Azure/azure-storage-fuse/blob/4ddfcd4b776650ae5172663c04db2a0fb791cbd6/cmd/mount.go#L129\n\nRequest:\n\nPlease add support for either:\n\n- --cleanup-on-start\n- --empty-dir-check\nas BlobFuse2 CLI parameters for the block_cache component, so that users relying on CLI-only configuration (e.g., in Azure Batch) can ensure a clean startup after reboots.\n\nThis would enable more robust and fault-tolerant behavior in environments where BlobFuse2 is managed non-interactively.\n\nThank you!", "author": "srxmsft", "created_at": "2025-05-16T05:19:36+00:00", "comments": [{"author": "srxmsft", "comment": "@syeleti-msft @vibhansa-msft \nAny suggestion on this?"}, {"author": "syeleti-msft", "comment": "Sure, this can be added. Thanks for the suggestion."}]} +{"id": "issue-1763", "type": "issue", "title": "fix CVE-2025-22871", "body": "Hi Team,\n\nThis is about the CVE-2025-22871 which is critical 9.1 base on NVD 3.x. \n\nHere is [GO-2025-3563](https://pkg.go.dev/vuln/GO-2025-3563) vulnerability details. \n\nCan you please help on fixing this.", "author": "sravansagi", "created_at": "2025-05-15T10:25:29+00:00", "comments": [{"author": "sravansagi", "comment": "Thanks for taking care of this"}]} +{"id": "issue-1728", "type": "issue", "title": "List after mount latency", "body": "What happened: Mounted folder intermittently takes multiple seconds for the files to appear. I am mounting an Azure blob container using blob fuse.\n\nWhat you expected to happen: Files to appear directly after mounting.\n\nAnything else we need to know?:\n\n] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only false, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 120, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace false, extension , disable-writeback-cache false, dirPermission 511, mountPath \nApr 28 18:10:40 aks-nodepool1-example-example blobfuse2[1063735]: \n\n[/var/lib/kubelet/plugins/kubernetes.io/csi/blob.csi.azure.com/d712ee9020e4ce800389cf61ad0eb9e0a732f7ecc1c82b77647b23b84c5bf859/globalmount] LOG_CRIT [config.go (519)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\nApr 28 18:10:40 aks-nodepool1-example-vmss000291 blobfuse2[1063735]: [/var/lib/kubelet/plugins/kubernetes.io/csi/blob.csi.azure.com/d712ee9020e4ce800389cf61ad0eb9e0a732f7ecc1c82b77647b23b84c5bf859/globalmount] LOG_CRIT [config.go (520)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\nApr 28 18:10:40 aks-nodepool1-26141071-exampleblobfuse2[1063735]: [/var/lib/kubelet/plugins/kubernetes.io/csi/blob.csi.azure.com/d712ee9020e4ce800389cf61ad0eb9e0a732f7ecc1c82b77647b23b84c5bf859/globalmount] LOG_CRIT [config.go (523)]: ParseAndValidateConfig : Telemetry : , honour-ACL false\nApr 28 18:10:41 aks-nodepool1-example-example[1063735]: [/var/lib/kubelet/plugins/kubernetes.io/csi/blob.csi.azure.com/d712ee9020e4ce800389cf61ad0eb9e0a732f7ecc1c82b77647b23b84c5bf859/globalmount] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob poc/example/*.csv does not exist\n\n\nEnvironment:\n\nCSI Driver version:Driver Version: v1.24.9\nBlobfuse2 Mount : 2.4.1 on [Ubuntu 22.04.5 LTS]\n\nAsk] Is it expected to wait for 5-10 seconds after mounting for the files to appear?", "author": "mahmoudsamy27", "created_at": "2025-05-04T11:21:51+00:00", "comments": [{"author": "vibhansa-msft", "comment": "It's by design, for first 5 seconds we do not allow list to go through. This is save some cost at customer end. Generally as soon as you mount, linux kernel will try to list the files/directories in the mounted path. If this call goes through then blobfuse will end up sending a listBlob call to storage container, which will cost the user as every REST call is charged. If customer is ok to take the cost then you can disable this feature. Be aware if you have millions of files in container then this will not only cost you but also create a latency as the listing will take some time as well."}, {"author": "vibhansa-msft", "comment": "In case you decide to disable this use \"--cancel-list-on-mount-seconds=0\" CLI option in your mount command."}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update on this item. If the suggested change did not fix the issue for you, feel free to comment and provide more inputs here."}]} +{"id": "issue-1724", "type": "issue", "title": "Stale file in blobfuse2 after mounting", "body": "**Which version of blobfuse was used?**\nBlobfuse2 2.5.0~preview.1\n\n**Which OS distribution and version are you using?**\nUbuntu,\n20.04.5 LTS\n\n**If relevant, please share your mount command.**\n/usr/bin/blobfuse2 mount ${MNTDIR} --container-name=$1 --attr-cache-timeout=0 --config-file=$CONFIG\n\n**What was the issue encountered?**\nFile is \"stuck\" with the first version after mounting. Aterwards, it does not update to any newer version coming from the storage account.\n\n**Have you found a mitigation/solution?**\nSo far, only either 1) remount or 2) sudo sysctl -w vm.drop_caches=3 can solve it\n\n**config.yaml**\n\nallow-other: true\nlogging:\n type: base\n level: log_debug\n file-path: /home/username/mount_log/data.log\nlibfuse:\n attribute-expiration-sec: 20\n entry-expiration-sec: 240\nazstorage:\n type: adls\n account-name: xx123xx\n endpoint: https://xx123xx.dfs.core.windows.net\n mode: msi\n update-md5: false\n validate-md5: false\n virtual-directory: true\n max-results-for-list: 0\n disable-compression: false\n telemetry: \"\"\n honour-acl: false\n cpk-enabled: false\n cpk-encryption-key: \"\"\n cpk-encryption-key-sha256: \"\"\n preserve-acl: false\n filter: \"\"\n user-assertions: \"\"\ncomponents:\n - libfuse\n - azstorage\n\n\n**Please share logs if available.**\n\nBelow are the operations carried out: after a fresh mount. First ls the file sss.txt. \n-rw-r----- 1 username username 74 May 1 16:18 sss.txt\n\nThen go to another machine and modify the file. Save, I can verify that the file is now\n-rw-r----- 1 username username 84 May 1 17:01 sss.txt\n\nBack to the original machine, the file stays stale unless I remount or flush kernel by \"sudo sysctl -w vm.drop_caches=3\". Logs excerpt are the following. It's able to see that the file size is now 84 instead of 74, but does not download it.\n\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_DEBUG [utils.go (162)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/6.953398ms, OpTime=6.975498ms) -- RESPONSE RECEIVED\n HEAD https://xx123xx.blob.core.windows.net/synapse-data/AllData%2Fbarra%2Fusfast%2Fbackup%2Ftest_estu%2Fsss.txt\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.5.0~preview.1 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.6.1 (go1.24.0; linux)\n x-ms-version: 2025-05-05\n \n--------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Accept-Ranges: bytes\n Content-Length: 84\n Content-Md5: REDACTED\n Content-Type: application/octet-stream\n Date: Thu, 01 May 2025 21:04:11 GMT\n Etag: \"0x8DD88F349B28B7C\"\n Last-Modified: Thu, 01 May 2025 21:01:05 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n X-Ms-Access-Tier: Hot\n X-Ms-Access-Tier-Inferred: true\n X-Ms-Acl: user::rw-,group::r--,other::---\n X-Ms-Blob-Type: BlockBlob\n X-Ms-Creation-Time: Wed, 30 Apr 2025 19:43:24 GMT\n X-Ms-Group: $superuser\n X-Ms-Lease-State: available\n X-Ms-Lease-Status: REDACTED\n X-Ms-Owner: 442b2576-f515-447a-9900-c8311206e92b\n X-Ms-Permissions: rw-r-----\n X-Ms-Request-Id: 0c03c99f-401e-004c-0edc-ba429f000000\n X-Ms-Resource-Type: file\n X-Ms-Server-Encrypted: REDACTED\n X-Ms-Version: 2025-05-05\n\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_DEBUG [utils.go (162)]: SDK(Retry) : response 200\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_DEBUG [utils.go (162)]: SDK(Retry) : exit due to non-retriable status code\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_TRACE [libfuse_handler.go (728)]: Libfuse::libfuse_open : AllData/barra/usfast/backup/test_estu/sss.txt, handle 7\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_TRACE [libfuse_handler.go (805)]: Libfuse::libfuse_flush : AllData/barra/usfast/backup/test_estu/sss.txt, handle: 7\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_TRACE [libfuse_handler.go (861)]: Libfuse::libfuse_release : AllData/barra/usfast/backup/test_estu/sss.txt, handle: 7\nThu May 1 17:04:11 EDT 2025 : blobfuse2[29476] : [/mcmnt/synapse-data] LOG_TRACE [azstorage.go (412)]: AzStorage::CloseFile : AllData/barra/usfast/backup/test_estu/sss.txt", "author": "surrealmemoir", "created_at": "2025-05-01T21:11:00+00:00", "comments": [{"author": "vibhansa-msft", "comment": "If doing 1sudo sysctl -w vm.drop_caches=3` solves the issue for you then it means kernel has cached the contents and requests are not even reaching to blobfuse. To configure blobfuse in a way where kernel is not in caching mode use \"-o direct_io\" option on your mount cli. But be-aware that this means neither kernel nor Blobfuse will have any cache and may result into higher network calls to backend storage and higher billing due to same."}, {"author": "surrealmemoir", "comment": "The issue is that I'm seeing the file stay stale long after the various timeout parameters are passed. I'm wondering why is the kernel cached \"locked\". \n\nIf we don't use direct_io, do you think the following cache timeout parameters should ideally all match?\n\nlibfuse.attribute-expiration-sec\nlibfuse.entry-expiration-sec\nfile_cache.timeout-sec\nattr_cache.timeout-sec\n\nI'm starting to suspect that if these are not the same, an ls call may trigger a meta update but not a file content update, thus \"locking\" the file to be a stale state, perhaps with an updated meta file info but outdated file content."}, {"author": "vibhansa-msft", "comment": "None of the cache parameters control how long kernel cache will keep the data, so only way around is to keep manually clearing the kernel cache."}, {"author": "vibhansa-msft", "comment": "Closing this as this is all by design. Feel free to post here in case you have any further queries on this."}]} +{"id": "issue-1687", "type": "issue", "title": "rmdir succeeds on non-empty directories when using blobfuse2 v2.4.0+", "body": "### Environment:\n\n**OS:** Debian 11\n**blobfuse2 version:** 2.4.0+ (reproducible on latest release)\n**Works correctly on:** 2.3.2 and below\n**Filesystem:** Mounted blob container via blobfuse2\n**Mount configuration:** `blobfuse2 mount all /data --config-file=fuse2_connection.yml`\n**File Cache mount configuration:** via _rc.local:_ `mount -t tmpfs -o size=2g tmpfs /tmp_blobfuse`\n**Azure Storage:** StorageV2 (general purpose v2), HNS Disabled\n\n### Bug Description:\n\nIt appears that since blobfuse2 version 2.4.0, it is possible to delete a blobfuse2 directory _(rmdir)_ even if it is not truly empty, as long as the files have been cleared from the blobfuse cache.\n\nThis is a critical issue for us. In our application logic, we delete files and then recursively attempt to delete parent directories only if they are empty,:\n```\ndelete file\ntry to delete parent directory (Filesystem will block if not empty)\nif success\ntry to delete parent's parent directory (Filesystem will block if not empty)\n\ncombined with a blobfuse2 issue is the problem.\n``` \n\nNormally, the filesystem correctly blocks deletion of non-empty directories. However, due to this blobfuse2 behavior since 2.4.0, _rmdir_ succeeds even when the directory still contains files, resulting file deletion ends up deleting the entire directory (and all files in it) instead of just the target file.\n\nDowngrading to blobfuse2 v2.3.2 or earlier eliminates the issue, confirming it was introduced since v2.4.0 (Since Dec. 2024).\n\n**Expected Behavior:**\nrmdir should fail if the target directory still contains blobs, regardless of blobfuse's local cache state.\n\n**Actual Behavior:**\nAfter file deletion and cache expiration, rmdir succeeds even though the corresponding blobs still exist in the mounted container.\n\n**Reproduction Script:**\nBelow is a minimal Bash script we use to reproduce the issue:\n```\n#!/bin/bash\n\nnumFiles=4\nmountPoint=\"/data/files\"\ncacheDir=\"/tmp_blobfuse/files\"\n\nnewDir=`head -c8 /dev/urandom | xxd -ps`\n\nblobfuse2 version\necho\n\n\necho \"Creating new directory $mountPoint/$newDir\"\nmkdir -p \"$mountPoint/$newDir\"\n\necho \"Creating $numFiles 128-byte files in directory\"\nfor i in $( seq 1 $numFiles ); do\n\thead -c128 /dev/urandom > \"$mountPoint/$newDir/file$i\"\ndone\n\necho\ndate\necho \"Number of files in directory: `ls -1 \"$mountPoint/$newDir/\" | wc -l`\"\necho \"Number of files in cache: `ls -1 \"$cacheDir/$newDir/\" | wc -l`\"\n\nrmdir \"$mountPoint/$newDir\"\nif [ $? -eq 0 ]; then\n\techo \"rmdir SUCCESS - This is unexpected!\"\nelse\n\techo \"rmdir FAILED - OK\"\nfi\n\necho\necho \"Wait for 4 minutes...\"\nsleep 60\necho \"3 more minutes...\"\nsleep 60\necho \"2 more minutes...\"\nsleep 60\necho \"Just one more minute...\"\nsleep 60\n\necho\ndate\necho \"Number of files in directory: `ls -1 \"$mountPoint/$newDir/\" | wc -l`\"\necho \"Number of files in cache: `ls -1 \"$cacheDir/$newDir/\" | wc -l`\"\n\nrmdir \"$mountPoint/$newDir\"\nif [ $? -eq 0 ]; then\n\techo \"rmdir SUCCESS - Regardless of directory not empty!\"\nelse\n\techo \"rmdir FAILED - OK\"\nfi\n``` \n\n\n**YAML Configuration:**\n```\nallow-other: true\nnonempty: true\nlogging:\n type: syslog\nlibfuse:\n attribute-expiration-sec: 240\n entry-expiration-sec: 240\n negative-entry-expiration-sec: 120\nfile_cache:\n path: /tmp_blobfuse\n timeout-sec: 120\nattr_cache:\n timeout-sec: 1800\nazstorage:\n account-name: xxxxxxxxxxxxxxxx\n account-key: xxxxxxxxxxxxxxxx\n endpoint: xxxxxxxxxxxxxxxx\n update-md5: false\n validate-md5: false\n virtual-directory: true\n max-results-for-list: 0\n disable-compression: false\n telemetry: \"\"\n honour-acl: false\n cpk-enabled: false\n cpk-encryption-key: \"\"\n cpk-encryption-key-sha256: \"\"\nmountall:\n container-allowlist:\n - storage\n - backups\n - files\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\n```", "author": "topcon-xchai", "created_at": "2025-04-14T19:47:32+00:00", "comments": [{"author": "vibhansa-msft", "comment": "There was a change to remove local empty directories and I see that has caused this regression. We will fix this in our next release."}, {"author": "topcon-xchai", "comment": "Thanks, @vibhansa-msft! Do you have an estimated timeline for the 2.5.0 release?"}, {"author": "vibhansa-msft", "comment": "No fixed plan yet but mostly it shall happen somewhere in next month."}]} +{"id": "issue-1683", "type": "issue", "title": "Blobfuse2 Data Inconsistency in AKS", "body": "We are using Blobfuse2 in AKS 1.31.4. CSI driver 1.25.3.\n\n**Description:** We are experiencing inconsistencies with Blobfuse even though we have disabled all forms of cache. Our cluster is appending to the same file, but when appending, we do not get the latest version of the file. The file can take up to 1 minute to update.\n\n**Steps to Reproduce:**\n1. Disable all forms of cache in Blobfuse.\n2. Append data to a file in a cluster environment.\n3. Attempt to read the latest version of the file immediately after appending.\n\n**Expected Behavior:** The file should reflect the latest appended data immediately.\n\n**Actual Behavior:** The file takes up to 1 minute to update and reflect the latest appended data.\n\n**We added this to the mount options:**\n```\nFuse options\n--attr-timeout=0\n--entry-timeout=0\n--negative-timeout=0\n--disable-writeback-cache=true\n\nFile cache options\n--file-cache-timeout=0\n```", "author": "al01101001k", "created_at": "2025-04-11T10:57:06+00:00", "comments": [{"author": "andyzhangx", "comment": "Why am I not able to see the updated contents of file(s), which were updated through means other than Blobfuse2 mount? If your use-case involves updating/uploading file(s) through other means and you wish to see the updated contents on Blobfuse2 mount then you need to disable kernel page-cache. -o direct_io CLI parameter is the option you need to use while mounting. Along with this, set file-cache-timeout=0 and all other libfuse caching parameters should also be set to 0. User shall be aware that disabling kernel cache can result into more calls to Azure Storage which will have cost and performance implications.\n\nhttps://github.com/Azure/azure-storage-fuse?tab=readme-ov-file#frequently-asked-questions"}, {"author": "mortenjoenby", "comment": "@andyzhangx , are you saying that \"-o direct_io\" disables kernel page-cache? Or is something else required to do that?"}, {"author": "andyzhangx", "comment": "> -o direct_io\n\nyes, use `-o direct_io` should work"}, {"author": "vibhansa-msft", "comment": "Closing this as there is no action item open on blobfuse. Feel free to reopen or post your comments here if you have any further queries."}]} +{"id": "issue-1679", "type": "issue", "title": "Changes made to files in container is not reflecting in VM", "body": "blobefuse2 version: 2.4.0\n\nOS distribution: Ubuntu 22.04\n\n\nIssue encountered:\nI have a VM where I'm using Blobfuse2 to mount a container from an Azure Storage account. After mounting, I'm able to view, edit, and create files from the VM, and any changes I make are successfully reflected in the blob container.\n\nIn my scenario, other applications also update files directly in the same container via the storage account. The issue is that changes made to existing files by these external applications are not reflected in the Blobfuse mount on the VM. But, any new files created by these applications do show up in the mount, but modifications to existing files do not.\n\nI\u2019ve tried setting the cache expiration time to `0`, but it didn\u2019t resolve the problem. \n\n\nConfig.yaml:\n\n```\nlogging:\n level: log_debug\n file-path: /var/log/blobfuse2-ws.log\n type: base\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 120\nfile_cache:\n path: /mnt/blobfuse-ws-cache\n timeout-sec: 120\n tmp-path: /mnt/blobfuse-ws-tmp\nattr_cache:\n timeout-sec: 120\nazstorage:\n account-name: xxx\n container: xxx\n type: adls\n mode: msi\n```", "author": "VijayaraghavanPS", "created_at": "2025-04-09T10:29:01+00:00", "comments": [{"author": "vibhansa-msft", "comment": "If there are other mount points which are updating the file or creating new files, and you wish to reflect back the latest changes then you need to mount with \"-o direct_io\" option. Be aware that this option will disable all sort of caching and will incur more calls to storage. Which also means higher cost and latency or degraded performance."}, {"author": "VijayaraghavanPS", "comment": "Thank you for the quick response. Should I use -o direct_io as a cli parameter or should I use it in the config file like direct_io = true"}, {"author": "vibhansa-msft", "comment": "Both have the same effect, easier is to just add \"-o direct_io\" in your mount command (cli parameter)"}, {"author": "VijayaraghavanPS", "comment": "Thanks. The sync is working fine after adding \"-o direct_io\".\nI would also like to understand how this will affect caching. \nI have set my config. yaml as\n```\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 120\nfile_cache:\n path: /mnt/blobfuse-ws-container-cache\n timeout-sec: 120\n tmp-path: /mnt/blobfuse-ws-container-tmp\nattr_cache:\n timeout-sec: 120\n```\n\nI edited a file in container. Then opened it immediately in the VM and I can see the changes. Is \"-o direct_io\" it disable all cache by overwriting the config values or is it in effect only when a file is opened."}, {"author": "vibhansa-msft", "comment": "When you set direct_io option, all the caching setting are ignored and it will disable all kernel and blobfuse caching."}, {"author": "VijayaraghavanPS", "comment": "Is it possible to retain the attr_cache (to reduce number of list operations) but still want to sync the changes. The changes don't need to sync immediately."}, {"author": "vibhansa-msft", "comment": "If you retain the attr-cache then last modified file time and file size does not get reflected and system will keep assuming the file contents have not changed and it may not work as you want it to work."}]} +{"id": "issue-1676", "type": "issue", "title": "Blobfuse2 Retains Cache for Large Files, Causing Failures on Subsequent Uploads", "body": "We have been using Blobfuse2 in our production environment for over a year. However, over the past month, we have encountered an issue where cache files for very large files are not being cleared properly. This results in failures when the source system attempts to upload a file with the same name the following day.\n\nThe issue is resolved only after manually deleting the cached large file, which introduces instability and manual intervention in our production workflows.\n\nPlease investigate and advise on a potential fix or configuration to handle this scenario more gracefully.\n\n\nConnection YAML:\n\nlogging:\n\n type: syslog\n\n level: log_debug\n\n \n\ncomponents:\n\n - libfuse\n\n - file_cache\n\n - attr_cache\n\n - azstorage\n\n \n\nlibfuse:\n\n attribute-expiration-sec: 0\n\n entry-expiration-sec: 0\n\n negative-entry-expiration-sec: 0\n\n disablewritebackcache: true\n\n ignoreopenflag: false\n\n \n\nfile_cache:\n\n path: /mnt/datadiskcache\n\n allow-non-empty-temp: true\n\n \n\nattr_cache:\n\n timeout-sec: 60\n\n \n\n \n\nazstorage:\n\n type: adls\n\n account-name: ***\n\n endpoint: [https://***.dfs.core.windows.net](https://%2A%2A%2A.dfs.core.windows.net/)\n\n mode: MSI\n\n container: outbound", "author": "kalirajann", "created_at": "2025-04-04T12:54:11+00:00", "comments": [{"author": "vibhansa-msft", "comment": "```\nfile_cache:\n\npath: /mnt/datadiskcache\n\nallow-non-empty-temp: true\n```\nAs per your file-cache config the files, once cached, will remain in cache for the default timeout of 120 seconds. Post this if the file is not in use, will get evicted from the local cache. Kindly ensure from your application end that file handles are closed properly after the use. If handle remains open file will not be evicted from cache. You can enable log debug and then validate why a particular file fails to be evicted."}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update on the issue. Feel free to reopen if you have any further queries on this."}]} +{"id": "issue-1672", "type": "issue", "title": "backup of files to blob storage get truncated", "body": "Hi,\nI am tring to use rman to make a backup of backup pieces to blob storage as a seconfairy backup location.\nFrom the rman command all goes ok, but when checking the blobfuse logging I see files get truncated.\nAnd do not show the correct size on the blob stoage.\nFrom the logging:\n\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02 does not exist\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd03cw_.bkp does not exist\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd03d4_.bkp does not exist\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd03d4_.bkp does not exist\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd03d4_.bkp [no such file or directory]\nWed Apr 2 14:39:31 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd03d4_.bkp does not exist\nWed Apr 2 14:39:34 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd06cl_.bkp does not exist\nWed Apr 2 14:39:34 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd06ct_.bkp does not exist\nWed Apr 2 14:39:34 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd06ct_.bkp does not exist\nWed Apr 2 14:39:34 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd06ct_.bkp [no such file or directory]\nWed Apr 2 14:39:34 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd06ct_.bkp does not exist\nWed Apr 2 14:39:41 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0fd9_.bkp does not exist\nWed Apr 2 14:39:41 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0ff4_.bkp does not exist\nWed Apr 2 14:39:41 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0ff4_.bkp does not exist\nWed Apr 2 14:39:41 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0ff4_.bkp [no such file or directory]\nWed Apr 2 14:39:41 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0ff4_.bkp does not exist\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02 does not exist\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02/o1_mf_nnnnf__mytd0ngw_.bkp does not exist\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02/o1_mf_nnnnf__mytd0nh4_.bkp does not exist\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02/o1_mf_nnnnf__mytd0nh4_.bkp does not exist\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02/o1_mf_nnnnf__mytd0nh4_.bkp [no such file or directory]\nWed Apr 2 14:39:48 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/F666881256D5E568E053E40110AC22B3/backupset/2025_04_02/o1_mf_nnnnf__mytd0nh4_.bkp does not exist\nWed Apr 2 14:39:50 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0p9k_.bkp does not exist\nWed Apr 2 14:39:50 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0p9s_.bkp does not exist\nWed Apr 2 14:39:50 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0p9s_.bkp does not exist\nWed Apr 2 14:39:50 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0p9s_.bkp [no such file or directory]\nWed Apr 2 14:39:50 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0p9s_.bkp does not exist\nWed Apr 2 14:39:53 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0shf_.bkp does not exist\nWed Apr 2 14:39:53 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0sho_.bkp does not exist\nWed Apr 2 14:39:53 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0sho_.bkp does not exist\nWed Apr 2 14:39:53 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0sho_.bkp [no such file or directory]\nWed Apr 2 14:39:53 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd0sho_.bkp does not exist\nWed Apr 2 14:40:00 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd106b_.bkp does not exist\nWed Apr 2 14:40:00 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd106n_.bkp does not exist\nWed Apr 2 14:40:00 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd106n_.bkp does not exist\nWed Apr 2 14:40:00 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd106n_.bkp [no such file or directory]\nWed Apr 2 14:40:00 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd106n_.bkp does not exist\nWed Apr 2 14:40:07 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd1759_.bkp does not exist\nWed Apr 2 14:40:07 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd175n_.bkp does not exist\nWed Apr 2 14:40:07 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd175n_.bkp does not exist\nWed Apr 2 14:40:07 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd175n_.bkp [no such file or directory]\nWed Apr 2 14:40:07 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd175n_.bkp does not exist\nWed Apr 2 14:40:09 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd19s5_.bkp does not exist\nWed Apr 2 14:40:09 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd19yw_.bkp does not exist\nWed Apr 2 14:40:10 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd19yw_.bkp does not exist\nWed Apr 2 14:40:10 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_ERR [block_blob.go (1216)]: BlockBlob::TruncateFile : Failed to get attributes of file TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd19yw_.bkp [no such file or directory]\nWed Apr 2 14:40:10 CEST 2025 : blobfuse2[731826] : [/mnt/oracledbtest001] LOG_WARNING [block_blob.go (510)]: BlockBlob::getAttrUsingList : blob TESTCDB/backup/TESTCDB/backupset/2025_04_02/o1_mf_nnnnf__mytd19yw_.bkp does not exist\n\nWil paste my config.yaml.", "author": "jasperkle", "created_at": "2025-04-02T12:48:56+00:00", "comments": [{"author": "jasperkle", "comment": "# General parameters\nallow-other: true\n\nlogging:\n type: base\n level: log_warning\n# level: LOG_TRACE\n file-path: /blobcache/log/blobfuse2.log\n\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 240\n entry-expiration-sec: 240\n negative-entry-expiration-sec: 240\n\nfile_cache:\n path: /blobcache/tmp\n timeout-sec: 14400\n max-size-mb: 307200\n\nattr_cache:\n timeout-sec: 120\n\nazstorage:\n type: block\n account-name: xxx\n account-key: xxx\n mode: key\n container: xxx"}, {"author": "vibhansa-msft", "comment": "As per logs that you have shared above, for most files we are getting an error of \"file not found\". If you are uploading contents to a fresh container on storage account, this is expected as there are no files initially and before copying/creating a file we will check its existence on the container. As file is not present in the container, we will get a 404.\nBy \"truncated\" you mean, your files are not of actual size post the backup operation or you are just worried about the logs?"}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update. Feel free to reopen if issue persist or you have any further queries on this."}]} +{"id": "issue-1662", "type": "issue", "title": "Question about the file /tmp/blobfuse2.pid", "body": "A cx reported one issue that one file /tmp/blobfuse2.pid(xxxxx) took 10GB size and it caused the /tmp folder out of space.\n\nIn mount.go, \n\n\t\tlog.Info(\"mount: Mounting blobfuse2 on %s\", options.MountPath)\n\t\tif !options.Foreground {\n\t\t\tpidFile := strings.Replace(options.MountPath, \"/\", \"_\", -1) + \".pid\"\n\t\t\tpidFileName := filepath.Join(os.ExpandEnv(common.DefaultWorkDir), pidFile)\n\n\t\t\tpid := os.Getpid()\n\t\t\tfname := fmt.Sprintf(\"/tmp/blobfuse2.%v\", pid)\n\n\t\t\tdmnCtx := &daemon.Context{\n\t\t\t\tPidFileName: pidFileName,\n\t\t\t\tPidFilePerm: 0644,\n\t\t\t\tUmask: 022,\n\t\t\t\tLogFileName: fname, // this will redirect stderr of child to given file\n\t\t\t}\n\nCan anyone explain how can I generate a /tmp/blobfuse2.pid with an error?", "author": "yangchengs", "created_at": "2025-03-20T09:40:16+00:00", "comments": [{"author": "syeleti-msft", "comment": "The /tmp/blobfuse2.pid file gets deleted as soon as the mount is successful,\nCould you please share the debug logs. \nyou can enable the logging by pasting the following at the top of your blobfuse2 configuration file. \n```\nlogging:\n type: base\n level: log_debug\n file-path: ~/path/to/log/file\n```\nQuestions:\n1. is the mount got successful when this happened?"}, {"author": "syeleti-msft", "comment": "Closing this issue as removing the fuse-trace option from the config would not increase the file size."}]} +{"id": "issue-1661", "type": "issue", "title": "blobfuse2 mount fails with invalid storage API version on AKS", "body": "I'm seeing blobfuse2 mount failures that started happening in a single of multiple similarily configured AKS clusters. The issue started to occur (or at least became visible) after upgrading AKS nodes images.\n\nAll nodes, including newly provisioned ones, in the affected cluster exhibit the same behaviour, i.e. mounting fails.\n\nThe storage account and container exists and the workload identity being used has the \"Storage Blob Data Contributor\" role.\n\nI inspected one of the nodes:\n\n```\n$ blobfuse2 -v\nblobfuse2 version 2.4.1\n\n$ uname -a\nLinux aks-nodepool-00000000-vmss00000E 5.15.173.1-2.cm2 #1 SMP Fri Feb 7 02:18:38 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux\n\n$ mount | grep blobfuse | uniq\n# none...\n```\n\nThe logs from `/var/log/blobfuse2.log appear to indicate that the storage API rejects requests because of an invalid header value. It is unclear to me, why blobfuse2 decides to report an authentication failure subsequently (that is also exposed to the outside).\n\n```\n2025-03-19T11:50:10.828082+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\n2025-03-19T11:50:10.827755+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\n2025-03-19T11:50:11.298632+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_WARNING [mount.go (418)]: mount: unsupported v1 CLI parameter: pre-mount-validate is always true in blobfuse2.\n2025-03-19T11:50:11.299016+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.4.1 on [CBL-Mariner/Linux]\n2025-03-19T11:50:11.299143+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_WARNING\n2025-03-19T11:50:11.299268+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only false, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 120, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace false, extension , disable-writeback-cache false, dirPermission 511, mountPath /var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount, umask 0\n2025-03-19T11:50:11.299424+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [file_cache.go (353)]: FileCache::Configure : create-empty false, cache-timeout 120, tmp-path /mnt/csi-dae73f1d6ec3dcad7764f32a3ef68177400fd2d662f3213ca08d770b7ee9f00b, max-size-mb 0, high-mark 80, low-mark 60, refresh-sec 0, max-eviction 5000, hard-limit false, policy , allow-non-empty-temp true, cleanup-on-start false, policy-trace false, offload-io false, sync-to-flush false, ignore-sync true, defaultPermission -rwxrwxrwx, diskHighWaterMark 0, maxCacheSize 24337.0875, mountPath /var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount\n2025-03-19T11:50:11.299572+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [attr_cache.go (162)]: AttrCache::Configure : cache-timeout 120, symlink false, max-files 5000000\n2025-03-19T11:50:11.299728+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (516)]: ParseAndValidateConfig : account stagingwmcblobs, container data-protection-keyring-staging, account-type BLOCK, auth KEY, prefix , endpoint https://my-storage-account.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\n2025-03-19T11:50:11.299895+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (519)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\n2025-03-19T11:50:11.300001+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (520)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\n2025-03-19T11:50:11.300242+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (523)]: ParseAndValidateConfig : Telemetry : , honour-ACL false\n2025-03-19T11:50:11.321839+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [block_blob.go (202)]: BlockBlob::TestPipeline : Failed to validate account with given auth %!s(func() string=0xc75f20)\n2025-03-19T11:50:11.322038+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (161)]: AzStorage::configureAndTest : Failed to validate credentials [GET https://my-storage-account.blob.core.windows.net/my-container#012--------------------------------------------------------------------------------#012RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.#012ERROR CODE: InvalidHeaderValue#012--------------------------------------------------------------------------------#012\ufeff#012InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.#012RequestId:0c8fa3ce-301e-00fa-7ac5-982ee7000000#012Time:2025-03-19T11:50:11.3236420Zx-ms-version2025-05-05#012--------------------------------------------------------------------------------#012]\n2025-03-19T11:50:11.322168+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (101)]: AzStorage::Configure : Failed to validate storage account [failed to authenticate credentials for azstorage]\n2025-03-19T11:50:11.322233+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [pipeline.go (82)]: Pipeline: error creating pipeline component azstorage [failed to authenticate credentials for azstorage]\n2025-03-19T11:50:11.322282+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [mount.go (458)]: mount : failed to initialize new pipeline [failed to authenticate credentials for azstorage]\n2025-03-19T11:50:11.298547+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_WARNING [mount.go (418)]: mount: unsupported v1 CLI parameter: pre-mount-validate is always true in blobfuse2.\n2025-03-19T11:50:11.298990+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.4.1 on [CBL-Mariner/Linux]\n2025-03-19T11:50:11.299123+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_WARNING\n2025-03-19T11:50:11.299245+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only false, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 120, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace false, extension , disable-writeback-cache false, dirPermission 511, mountPath /var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount, umask 0\n2025-03-19T11:50:11.299400+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [file_cache.go (353)]: FileCache::Configure : create-empty false, cache-timeout 120, tmp-path /mnt/csi-dae73f1d6ec3dcad7764f32a3ef68177400fd2d662f3213ca08d770b7ee9f00b, max-size-mb 0, high-mark 80, low-mark 60, refresh-sec 0, max-eviction 5000, hard-limit false, policy , allow-non-empty-temp true, cleanup-on-start false, policy-trace false, offload-io false, sync-to-flush false, ignore-sync true, defaultPermission -rwxrwxrwx, diskHighWaterMark 0, maxCacheSize 24337.0875, mountPath /var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount\n2025-03-19T11:50:11.299554+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [attr_cache.go (162)]: AttrCache::Configure : cache-timeout 120, symlink false, max-files 5000000\n2025-03-19T11:50:11.299706+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (516)]: ParseAndValidateConfig : account my-storage-account, container my-container, account-type BLOCK, auth KEY, prefix , endpoint https://my-storage-account.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\n2025-03-19T11:50:11.299872+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (519)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\n2025-03-19T11:50:11.299981+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (520)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\n2025-03-19T11:50:11.300216+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_CRIT [config.go (523)]: ParseAndValidateConfig : Telemetry : , honour-ACL false\n2025-03-19T11:50:11.321788+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [block_blob.go (202)]: BlockBlob::TestPipeline : Failed to validate account with given auth %!s(func() string=0xc75f20)\n2025-03-19T11:50:11.322019+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (161)]: AzStorage::configureAndTest : Failed to validate credentials [GET https://my-storage-account.blob.core.windows.net/my-container#012--------------------------------------------------------------------------------#012RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.#012ERROR CODE: InvalidHeaderValue#012--------------------------------------------------------------------------------#012\ufeff#012InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.#012RequestId:0c8fa3ce-301e-00fa-7ac5-982ee7000000#012Time:2025-03-19T11:50:11.3236420Zx-ms-version2025-05-05#012--------------------------------------------------------------------------------#012]\n2025-03-19T11:50:11.322157+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (101)]: AzStorage::Configure : Failed to validate storage account [failed to authenticate credentials for azstorage]\n2025-03-19T11:50:11.322226+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [pipeline.go (82)]: Pipeline: error creating pipeline component azstorage [failed to authenticate credentials for azstorage]\n2025-03-19T11:50:11.322275+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [mount.go (458)]: mount : failed to initialize new pipeline [failed to authenticate credentials for azstorage]\n```", "author": "jasper-d", "created_at": "2025-03-19T12:08:02+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "```\nLOG_ERR [block_blob.go (202)]: BlockBlob::TestPipeline : Failed to validate account with given auth %!s(func() string=0xc75f20)\n2025-03-19T11:50:11.322038+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (161)]: AzStorage::configureAndTest : Failed to validate credentials [GET https://my-storage-account.blob.core.windows.net/my-container#012--------------------------------------------------------------------------------#012RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.#012ERROR CODE: InvalidHeaderValue#012--------------------------------------------------------------------------------#012\ufeff#012InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.#012RequestId:0c8fa3ce-301e-00fa-7ac5-982ee7000000#012Time:2025-03-19T11:50:11.3236420Zx-ms-version2025-05-05#012--------------------------------------------------------------------------------#012]\n2025-03-19T11:50:11.322168+00:00 aks-nodepool-00000000-vmss00000E blobfuse2[334696]: [/var/lib/kubelet/pods/7ed2ded2-ea9b-4ab4-9b81-f168748420f9/volumes/kubernetes.io~csi/my-volume/mount] LOG_ERR [azstorage.go (101)]: AzStorage::Configure : Failed to validate storage account [failed to authenticate credentials for azstorage]\n\n```\nThis log clearly indicates something wrong with the authorization/authentication. Either some env variable is not exposed or some invalid format is there. Just to validate you can manually mount outside AKS with same config and check it works or not."}, {"author": "jasper-d", "comment": "Yes, I can reproduce this on a separate Azure VM using the same identity and blobfuse2:\n\n```\nMar 20 19:49:22 test-blobfuse blobfuse2[10596]: [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.5.0~preview.1 on [Ubuntu 24.04.2 LTS]\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_INFO [mount.go (439)]: Mount Command: [./blobfuse2 /tmp/blobfuse-2 --tmp-path=/tmp/blobfuse-1 --file-cache-timeout-in-seconds=4 --log-level=LOG_DEBUG --config-file=~/blobfuse.staging.yaml]\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_DEBUG\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [mount.go (441)]: Mount allowed on nonempty path : false\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [libfuse.go (284)]: Libfuse::Configure : libfuse\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_INFO [libfuse.go (252)]: Libfuse::Validate : UID 1000, GID 1000\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only false, allow-other false, allow-root false, default-perm 493, entry-timeout 10, attr-time 10, negative-timeout 10, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace false, extension , disable-writeback-cache false, dirPermission 509, mountPath /tmp/blobfuse-2, umask 0\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [file_cache.go (225)]: FileCache::Configure : file_cache\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [file_cache.go (353)]: FileCache::Configure : create-empty false, cache-timeout 4, tmp-path /tmp/blobfuse-1, max-size-mb 0, high-mark 80, low-mark 60, refresh-sec 0, max-eviction 5000, hard-limit false, policy , allow-non-empty-temp false, cleanup-on-start false, policy-trace false, offload-io false, sync-to-flush false, ignore-sync true, defaultPermission -rwxr-xr-x, diskHighWaterMark 0, maxCacheSize 18662.034375, mountPath /tmp/blobfuse-2\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [attr_cache.go (136)]: AttrCache::Configure : attr_cache\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [attr_cache.go (162)]: AttrCache::Configure : cache-timeout 10, symlink false, max-files 5000000\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [azstorage.go (84)]: AzStorage::Configure : azstorage\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [config.go (304)]: ParseAndValidateConfig : Parsing config\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_INFO [config.go (419)]: ParseAndValidateConfig : using the following proxy address from the config file:\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [config.go (566)]: ParseAndReadDynamicConfig : Reparsing config\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [config.go (426)]: ParseAndValidateConfig : Getting auth type\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [config.go (533)]: ParseAndValidateConfig : account my-storage-account, container data-protection-keyring-staging, account-type BLOCK, auth MSI, prefix , endpoint https://my-storage-account.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [config.go (536)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [config.go (537)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_CRIT [config.go (540)]: ParseAndValidateConfig : Telemetry : , honour-ACL false\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [block_blob.go (167)]: BlockBlob::SetupPipeline : Setting up\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [block_blob.go (148)]: BlockBlob::createServiceClient : Getting service client\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [azauth.go (89)]: azAuth::getAzAuth : Account: my-storage-account, AccountType: BLOCK, Protocol: https, Endpoint: https://my-storage-account.blob.core.windows.net/\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [block_blob.go (235)]: BlockBlob::SetPrefixPath : path\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_TRACE [block_blob.go (184)]: BlockBlob::TestPipeline : Validating\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://my-storage-account.blob.core.windows.net/data-protection-keyring-staging?comp=REDACTED&delimiter=REDACTED&maxresults=REDACTED&prefix=REDACTED&restype=REDACTED\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=REDACTED&resource=REDACTED\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=dc72e9c4-8493-40a6-9f5a-aced6ca44fa5&resource=https%3A%2F%2Fstorage.azure.com\n Metadata: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.2 (go1.24.0; linux)\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/6.311646ms, OpTime=6.344778ms) -- RESPONSE RECEIVED\n GET http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=dc72e9c4-8493-40a6-9f5a-aced6ca44fa5&resource=https%3A%2F%2Fstorage.azure.com\n Metadata: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.2 (go1.24.0; linux)\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Content-Length: 2000\n Content-Type: application/json; charset=utf-8\n Date: Thu, 20 Mar 2025 19:49:22 GMT\n Server: IMDS/150.870.65.1544\n X-Ms-Request-Id: e18d2433-8f28-4492-b778-9b8ed36930a5\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Authentication) : ManagedIdentityCredential.GetToken() acquired a token for scope \"https://storage.azure.com\"\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://my-storage-account.blob.core.windows.net/data-protection-keyring-staging?comp=list&delimiter=%2F&maxresults=2&prefix=&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.5.0~preview.1 (Ubuntu 24.04.2 LTS) azsdk-go-azblob/v1.6.1-beta.1 (go1.24.0; linux)\n x-ms-version: 2025-05-05\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/23.081812ms, OpTime=23.137075ms) -- RESPONSE RECEIVED\n GET https://my-storage-account.blob.core.windows.net/my-container?comp=list&delimiter=%2F&maxresults=2&prefix=&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.5.0~preview.1 (Ubuntu 24.04.2 LTS) azsdk-go-azblob/v1.6.1-beta.1 (go1.24.0; linux)\n x-ms-version: 2025-05-05\n --------------------------------------------------------------------------------\n RESPONSE Status: 400 The value for one of the HTTP headers is not in the correct format.\n Content-Length: 329\n Content-Type: application/xml\n Date: Thu, 20 Mar 2025 19:49:22 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n X-Ms-Request-Id: 49dffcf2-d01e-0080-78d1-9933a7000000\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 400\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_DEBUG [utils.go (156)]: SDK(ResponseError) : GET https://my-storage-account.blob.core.windows.net/my-container\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.\n ERROR CODE: InvalidHeaderValue\n --------------------------------------------------------------------------------\n \ufeff\n InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.\n RequestId:49dffcf2-d01e-0080-78d1-9933a7000000\n Time:2025-03-20T19:49:23.1938810Zx-ms-version2025-05-05\n --------------------------------------------------------------------------------\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [block_blob.go (203)]: BlockBlob::TestPipeline : Failed to validate account with given auth GET https://my-storage-account.blob.core.windows.net/my-container\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.\n ERROR CODE: InvalidHeaderValue\n --------------------------------------------------------------------------------\n \ufeff\n InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.\n RequestId:49dffcf2-d01e-0080-78d1-9933a7000000\n Time:2025-03-20T19:49:23.1938810Zx-ms-version2025-05-05\n --------------------------------------------------------------------------------\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [azstorage.go (161)]: AzStorage::configureAndTest : Failed to validate credentials [BlockBlob::TestPipeline : [InvalidHeaderValue]]\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [azstorage.go (101)]: AzStorage::Configure : Failed to validate storage account [failed to authenticate azstorage credentials with error [BlockBlob::TestPipeline : [InvalidHeaderValue]]]\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [pipeline.go (82)]: Pipeline: error creating pipeline component azstorage [failed to authenticate azstorage credentials with error [BlockBlob::TestPipeline : [InvalidHeaderValue]]]\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [mount.go (458)]: mount : failed to initialize new pipeline [failed to authenticate azstorage credentials with error [BlockBlob::TestPipeline : [InvalidHeaderValue]]]\n```\n\nWhile blobfuse2 fails to mount the container, I can access it with the same workload identity using both `az` and .NET `Azure.Storage.Blobs` client."}, {"author": "jasper-d", "comment": "@olehsluzhala pointed me to https://learn.microsoft.com/en-us/rest/api/storageservices/versioning-for-the-azure-storage-services#invalidheadervalue-error-message\n\nWe tried to downgrade to blobfuse2 v2.3.2 and can confirm that using that version we cannot reproduce the issue. I presume this is not an issue for this repository but rather for Azure support."}, {"author": "souravgupta-msft", "comment": "`InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.#012RequestId:0c8fa3ce-301e-00fa-7ac5-982ee7000000#012Time:2025-03-19T11:50:11.3236420Zx-ms-version2025-05-05`\n\n@andyzhangx, is the service version `2025-05-05` unsupported in AKS environment?"}, {"author": "andyzhangx", "comment": "> `InvalidHeaderValueThe value for one of the HTTP headers is not in the correct format.#012RequestId:0c8fa3ce-301e-00fa-7ac5-982ee7000000#012Time:2025-03-19T11:50:11.3236420Zx-ms-version2025-05-05`\n> \n> [@andyzhangx](https://github.com/andyzhangx), is the service version `2025-05-05` unsupported in AKS environment?\n\n@souravgupta-msft no idea, I think that's related to the service version supported by that storage account, storage team may know more about this error."}, {"author": "souravgupta-msft", "comment": "@jasper-d, can you please retry this with `key` based auth mode and let us know if you are getting the same error."}, {"author": "jasper-d", "comment": "Behaviour with key is the same as with MSI. Works with 2.3.2 and fails with current main because of `x-ms-version2025-05-05`"}, {"author": "souravgupta-msft", "comment": "The issue is not with the auth mode, rather with the service version `2025-05-05` not supported by your environment or storage account. Let me check with the backend team."}, {"author": "akerckhofs", "comment": "Having identical issues on nodes using `AKSUbuntu-2204gen2containerd-202503.13.0`. Nodes using `AKSUbuntu-2204gen2containerd-202412.04.0` still function correctly. These nodes are in the same cluster, so same `csi-blob-node` DS.\n\n`blobfuse2 -v` on `AKSUbuntu-2204gen2containerd-202503.13.0` nodes: 2.4.1\n`blobfuse2 -v` on `AKSUbuntu-2204gen2containerd-202412.04.0` nodes: 2.3.2"}, {"author": "vibhansa-msft", "comment": "```\nMar 20 19:49:23 test-blobfuse blobfuse2[10596]: [/tmp/blobfuse-2] LOG_ERR [block_blob.go (203)]: BlockBlob::TestPipeline : Failed to validate account with given auth GET https://my-storage-account.blob.core.windows.net/my-container\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 The value for one of the HTTP headers is not in the correct format.\n ERROR CODE: InvalidHeaderValue\n --------------------------------------------------------------------------------\n \ufeff\n```\nAs per this error, backend is not supporting this service-version. This means the storage account is hosted on a cluster which is not upgraded to the latest service version. There is nothing blobfuse can help you with in this case. We will create a backlog item (feature request) to provide a knob to override the service version for future use. Feel free to post any further comments/queries here."}]} +{"id": "issue-1656", "type": "issue", "title": "blobfuse2 health monitor logs are empty", "body": "I am facing weird issue with health monitor logs, below is my blobfuse2 config yml\n```\nallow-other: true\nlogging:\n type: syslog\n level: log_debug\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\nfile_cache:\n path: /mnt/cache-container-temp\n timeout-sec: 120\nattr_cache:\n timeout-sec: 7200\nazstorage:\n type: block\n account-name: accnt\n sas: ***\n endpoint: https://accnt.blob.core.windows.net/\n mode: sas\n container: cache-container\nhealth_monitor:\n enable-monitoring: true\n stats-poll-interval-sec: 10\n process-monitor-interval-sec: 30\n output-path: /__w/workspace/a/blobfuse2\n```\n\nCommand:\n```\nblobfuse2 mount $cache_folder_path --config-file=${{ env.WORKSPACE }}/az_storage_cache_config.yml --log-level LOG_DEBUG --log-file-path ${{ env.BLOBFUSE2_ARTIFACTS_PATH }}/blobfuse2.log\n\n```\nI am trying to ls into the directory `output-path` right after blobfuse2 mount command but I am not able to find health monitor log file, but when I archive the artifacts from `output-path` directory, I do see an empty log file with name `monitor_pid.json` along with blobfuse2 log.\n\nblobfuse2 log has below entry when I try to search `health` :\n```\nLOG_DEBUG [health-monitor.go (102)]: health-monitor : Options = [--pid=820 --stats-poll-interval-sec=10 --process-monitor-interval-sec=30 --output-path=/__w/workspace/a/blobfuse2 --cache-path=/mnt/cache-container-temp --max-size-mb=0]\nLOG_DEBUG [health-monitor.go (103)]: health-monitor : Starting health-monitor for blobfuse2 pid = 820\n```\n\nI am not sure if there should be more entries where it mentons it finished health monitor, but I dont see it in logs\n\nwe use blobfuse2 v2.4.0\n\n![Image](https://github.com/user-attachments/assets/dd009468-54db-453c-8458-f7794904dc1f)", "author": "HarshadDGhorpade-eaton", "created_at": "2025-03-10T10:47:50+00:00", "comments": [{"author": "souravgupta-msft", "comment": "Hi @HarshadDGhorpade-eaton. I am unable to replicate this issue with the config shared. Can you share the contents of the above `monitor_pid.json` file."}, {"author": "HarshadDGhorpade-eaton", "comment": "> Hi [@HarshadDGhorpade-eaton](https://github.com/HarshadDGhorpade-eaton). I am unable to replicate this issue with the config shared. Can you share the contents of the above `monitor_pid.json` file.\n\n@souravgupta-msft \n\n![Image](https://github.com/user-attachments/assets/4e9dea8e-12a7-465c-ac38-faa963db06cd)"}, {"author": "vibhansa-msft", "comment": "You health monitor logs are being redirected to \"/__w/workspace/a/blobfuse2\". Kindly check this directory exists, we have recently observed, if the directory does not exists then logs are not dumped to a log file."}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update. Feel free to reopen if the issue persists."}]} +{"id": "issue-1651", "type": "issue", "title": "blobfuse2 failed to authenticate credentials", "body": "I'm trying to mount my azure blob storage container in my ubuntu docker container.\nblobfuse2 version:\nWed Mar 5 17:10:42 UTC 2025 : blobfuse2[5898] : [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load. blobfuse2 version 2.5.0~preview.1\n\nI checked that Blobfuse2 is compatible with Ubuntu 24.04:\nDISTRIB_ID=Ubuntu\nDISTRIB_RELEASE=24.04\nDISTRIB_CODENAME=noble\nDISTRIB_DESCRIPTION=\"Ubuntu 24.04.1 LTS\"\nPRETTY_NAME=\"Ubuntu 24.04.1 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.1 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo\n\nMy config.yaml file is in root@12342:/# /etc/blobfuse2/config.yaml and it's contents are:\ncomponents:\n libfuse\n azstorage\n file_cache\nlibfuse:\n path: models/data/mount\nazstorage:\n account-name: ****\n container: ****\n mode: key\n account-key: ****\nfile_cache:\n path: models/data/tmp\n\nI'm running the command blobfuse2 mount ./models/data/mount --config-file=../etc/blobfuse2/config.yaml and getting error: Thu Mar 6 20:59:29 UTC 2025 : blobfuse2[6306] : [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load. Error: failed to initialize new pipeline [failed to authenticate credentials for azstorage]. i've confirmed that the account name, container, and key are all correct; see screenshots from where i'm getting them. i do see that this says, Anonymous Access Level = Private, but I've been able to access it from the Python package and via rclone locally. (Can't use rclone now. locally i'm on windows, but my docker container is ubuntu).\n\n![Image](https://github.com/user-attachments/assets/06ebcab2-2c31-4386-a646-2ecb7524c1d5)\n![Image](https://github.com/user-attachments/assets/cbef2a94-daad-4165-961e-fcde68fe9320)", "author": "scottschmidl", "created_at": "2025-03-06T21:26:37+00:00", "comments": [{"author": "vibhansa-msft", "comment": "Are you accessing public azure cloud or some other variant ?"}, {"author": "vibhansa-msft", "comment": "With every new release of Blobfuse we tend to upgrade our storage SDK as well and as part of that new service-version will come inherently and this shall work on most tenants on public cloud."}, {"author": "scottschmidl", "comment": "> Are you accessing public azure cloud or some other variant ?\n\nIt's a storage account on Azure Government, but rclone worked with no issues. Are you telling me that blobfuse2 doesn't have the same functionality?\n\nEDIT: i added the endpoint parameter to point to usgovcloudapi.net but now i get an error that I didn't get before: Error: failed to initialize new pipeline [config error in Pipeline [component file_cache is out of order]]"}, {"author": "scottschmidl", "comment": "> > Are you accessing public azure cloud or some other variant ?\n> \n> It's a storage account on Azure Government, but rclone worked with no issues. Are you telling me that blobfuse2 doesn't have the same functionality?\n> \n> EDIT: i added the endpoint parameter to point to usgovcloudapi.net but now i get and error that I didn't get before: Error: failed to initialize new pipeline [config error in Pipeline [component file_cache is out of order]]\n\nIs it possible to please get some more help? I haven't been able to resolve the issue yet.\nThanks"}, {"author": "vibhansa-msft", "comment": "@scottschmidl Blobfuse team is here to help. For Govt Cloud as you correctly guessed you need to define the correct endpoint as by default Blobfuse assumes \".blob.core.windows.net\" as the endpoint.\nFor the above error regarding \"file_cache out of order\" appears there is an issue in your config file. Kindly share your config file details here (removing the storage account details and credentials)."}, {"author": "scottschmidl", "comment": "> [@scottschmidl](https://github.com/scottschmidl) Blobfuse team is here to help. For Govt Cloud as you correctly guessed you need to define the correct endpoint as by default Blobfuse assumes \".blob.core.windows.net\" as the endpoint. For the above error regarding \"file_cache out of order\" appears there is an issue in your config file. Kindly share your config file details here (removing the storage account details and credentials).\n\nhi thanks for your support. my config.yaml looks like this:\ncomponents:\n libfuse\n azstorage\n file_cache\nlibfuse:\n path: models/data/mount\nazstorage:\n type: block\n account-name: ****\n container: ****\n mode: key\n account-key: ****\n endpoint: https://****.blob.core.usgovcloudapi.net/\nfile_cache:\n path: models/data/tmp"}, {"author": "scottschmidl", "comment": "> > [@scottschmidl](https://github.com/scottschmidl) Blobfuse team is here to help. For Govt Cloud as you correctly guessed you need to define the correct endpoint as by default Blobfuse assumes \".blob.core.windows.net\" as the endpoint. For the above error regarding \"file_cache out of order\" appears there is an issue in your config file. Kindly share your config file details here (removing the storage account details and credentials).\n> \n> hi thanks for your support. my config.yaml looks like this: components: libfuse azstorage file_cache libfuse: path: models/data/mount azstorage: type: block account-name: **** container: **** mode: key account-key: **** endpoint: https://****.blob.core.usgovcloudapi.net/ file_cache: path: models/data/tmp\n\n\nI rearranged my config.yaml to follow what's here: https://github.com/Azure/azure-storage-fuse/blob/main/setup/baseConfig.yaml\n\nbut now I see this error:\nError: Mon Mar 10 19:06:49 UTC 2025 : blobfuse2[70] : [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\nfuse: device not found, try 'modprobe fuse' first\nError: unable to start pipeline [failed to mount fuse]"}, {"author": "scottschmidl", "comment": "this appears to be working now. i had to add: --cap-add=SYS_ADMIN --device /dev/fuse to docker run"}]} +{"id": "issue-1649", "type": "issue", "title": "blobfuse2 in ubuntu docker containre", "body": "I'm trying to mount my azure blob storage container in my ubuntu docker container.\nblobfuse2 version:\nWed Mar 5 17:10:42 UTC 2025 : blobfuse2[5898] : [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load. blobfuse2 version 2.5.0~preview.1\n\nI checked that Blobfuse2 is compatible with Ubuntu 24.04:\nDISTRIB_ID=Ubuntu\nDISTRIB_RELEASE=24.04\nDISTRIB_CODENAME=noble\nDISTRIB_DESCRIPTION=\"Ubuntu 24.04.1 LTS\"\nPRETTY_NAME=\"Ubuntu 24.04.1 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.1 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo\n\nMy config.yaml file is in root@12342:/# /etc/blobfuse2/config.yaml and it's contents are:\nlogging:\n type: syslog\n level: LOG_INFO\ncomponents:\n libfuse\n file_cache\n azstorage\nlibfuse:\n path: models/data/mount\nazstorage:\n account_name: xxxx\n account_key: xxxx\n container: xxxx\nfile_cache:\n tmp-path: models/data/tmp\n\nthe goal is that I have images/labels in azure blob storage container and i'm trying to use a YAML with Ultralytics YOLO to run object detection models. This worked locally when I mounted with rclone, but locally is Windows11, while my docker container is Ubuntu. my model is run from root@12342:/app/models/model_manager/model_manager.py. I'm running the command blobfuse2 mount ./models/data/mount --config-file=../etc/blobfuse2/config.yaml and getting error:\nWed Mar 5 17:20:55 UTC 2025 : blobfuse2[5914] : [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load. Error: failed to initialize new pipeline [config error in file_cache error [tmp-path not set]]. but as you can see i have file_cache: tmp-path: models/data/tmp set within my file structure.", "author": "scottschmidl", "created_at": "2025-03-05T17:22:41+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "@scottschmidl for file_cache the attribute name is not `tmp-path`, it's only `path`\n`file_cache:\n path: models/data/tmp\n`"}, {"author": "scottschmidl", "comment": "ah thank you this worked. i also found this link: https://github.com/Azure/azure-storage-fuse/blob/main/setup/baseConfig.yaml.\n\nwhich told me that my account_key was also wrong and needed to be account-key."}]} +{"id": "issue-1647", "type": "issue", "title": "provide container not found error directly in blobfuse mount error when `--container-name` does not exist", "body": "Currently when we mount a non-existing container, it would report error \"failed to authenticate credentials for azstorage\", which is not quite clear error, unless I have opened the `/var/log/blobfuse2.log`, and then I found it's container not found error, it's better to provide container not found error directly in blobfuse mount error when `--container-name` does not exist, that would save customers lots of debugging time when we hit such scenario in AKS.\n\n```\nblobfuse2 test --container-name=cdpblobnonprodeus --tmp-path=/tmp/blobfuse -o allow_other --file-cache-timeout-in-seconds=120\n*** blobfuse2: A new version [2.4.1] is available. Consider upgrading to latest version for bug-fixes & new features. ***\nError: failed to initialize new pipeline [failed to authenticate credentials for azstorage]\n```", "author": "andyzhangx", "created_at": "2025-03-04T01:55:19+00:00", "comments": [{"author": "syeleti-msft", "comment": "Sure, that make sense"}, {"author": "andyzhangx", "comment": "@syeleti-msft thanks for the check, `failed to authenticate credentials for azstorage` error does not provide enough info, besides container not exist error, could blobfuse return error provide more meaningful error for other errors if possible, thanks."}, {"author": "syeleti-msft", "comment": "Sure, I tried to return the error code that come from the SDK when failure happens.\nPR: https://github.com/Azure/azure-storage-fuse/pull/1648"}]} +{"id": "issue-1645", "type": "issue", "title": "Mounted directory contents are not accessible in blobfuse2 2.4.x", "body": "Recently, we've encountered an issue where the contents of a mounted directory on a server are not accessible. When attempting to list the sub-folder contents, the directory can be listed once, but if the command is run again, it throws the error:\n\nls: cannot open directory '.': Input/output error.\n\nUpon troubleshooting, we identified that the Blobfuse version was recently upgraded from 2.3.0 to 2.4.0 through Azure Update Manager. After rolling back the version to 2.3.0, the issue was resolved, and the directory became accessible again.\n\nCould someone please help us understand why this issue occurs with blobfuse2 version 2.4.0 and not with version 2.3.0?\n\n\"Image\"", "author": "SumanNakka", "created_at": "2025-02-26T21:33:43+00:00", "comments": [{"author": "vibhansa-msft", "comment": "Kindly share debug logs with 2.4.0 where directory listing fails, along with your mount command and config."}, {"author": "SumanNakka", "comment": "Thanks for the response, here are the requested,\n\nblobfuse2 mount /root/nalchbp-dev-sftpv3/ --config-file=/root/blobfuse2/blobfuse2.yaml\n\n[blobfuse2-dev.yaml.zip](https://github.com/user-attachments/files/19014760/blobfuse2-dev.yaml.zip)\n\n[blobfuse2-dev-debug-logs.zip](https://github.com/user-attachments/files/19014754/blobfuse2-dev-debug-logs.zip)"}, {"author": "syeleti-msft", "comment": "I tried to reproduce the issue, but I'm not able to do get the behavior that you are seeing. I would appreciate if you shared the steps to reproduce the behavior.\n\nAlso is it possible to verify storage account is not hns enabled. If it is, then you should change your config file likewise."}, {"author": "SumanNakka", "comment": "Hi, sure, here are the steps to reproduce,\n1. After mounting the directory on the VM using blobfuse, navigate inside the directory.\n2. Navigate to a sub-folder if there is any.\n3. List out the contents by ls -lrt or ls -la, and the contents will pop-up.\n4. If we try to list them out again and then we see the error.\n\nAlso, regarding the HNS, I don't see anything related to it. So, I'm assuming it was not enabled when the storage account was created."}, {"author": "syeleti-msft", "comment": "I am not able to reproduce,\n```\nfantom@syeleti-vm:~/mycontainer$ blobfuse2 --version\nblobfuse2 version 2.4.0\nfantom@syeleti-vm:~/mycontainer$ cd c2\nfantom@syeleti-vm:~/mycontainer/c2$ ls -lrt nalc_test\ntotal 0\n-rwxr-xr-x 1 fantom fantom 0 Mar 7 05:18 nalc_test\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 uploads\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 1095buploads\nfantom@syeleti-vm:~/mycontainer/c2$\nfantom@syeleti-vm:~/mycontainer/c2$ ls -lrt nalc_test\ntotal 0\n-rwxr-xr-x 1 fantom fantom 0 Mar 7 05:18 nalc_test\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 uploads\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 1095buploads\nfantom@syeleti-vm:~/mycontainer/c2$\nfantom@syeleti-vm:~/mycontainer/c2$ ls -lrt nalc_test\ntotal 0\n-rwxr-xr-x 1 fantom fantom 0 Mar 7 05:18 nalc_test\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 uploads\ndrwxrwxr-x 2 fantom fantom 4096 Mar 7 05:18 1095buploads\n```\nThe following are some things you can do to give us more context.\n1. Please send the debug logs for 2.3.0 and 2.4.x\n2. Please go to the Azure portal -> navigate to your container -> search for the blob nalc_test and share the result. Just wanted to know whether the directory blob is created or not.\n\n![Image](https://github.com/user-attachments/assets/bc941c75-99ba-4025-8865-38163185faa2)\n3. Modify stream to block_cache in your config.\n```\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n```"}, {"author": "fenglinnorway", "comment": "I am using blobfuse 2.4.1 and the configure file is like:\n\n```\n# Logger configuration\nlogging:\n type: syslog\n level: log_info\n file-path: /home/azureuser/\n\n# Pipeline configuration. Choose components to be engaged. The order below is the priority order that needs to be followed.\ncomponents:\n - libfuse\n - block_cache\n - file_cache\n - attr_cache\n - azstorage\n \n# Block cache related configuration\nblock_cache:\n path: /home/azureuser/tmp2\n\n# Disk cache related configuration\nfile_cache:\n # Required\n path: /home/azureuser/tmp2\n \n# Azure storage configuration\nazstorage:\n# Required\n account-key: xxx==\n type: block\n account-name: \n container: \n endpoint: https://xxxx.windows.net\n mode: key\n sas: \n```\n\n\ni also got the input/output error and the logs below:\n\n```\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1042)]: BlockCache::download : 3 attempts to download a block have failed 128=>20u/hgt_20u_001_000.csv (index 0, offset 0)\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (743)]: BlockCache::getBlock : Failed to download block 0 for 128=>20u/hgt_20u_001_000.csv (read offset 0)\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (594)]: BlockCache::ReadInBuffer : Failed to get Block 128=>20u/hgt_20u_001_000.csv offset 0 [failed to download block]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [libfuse_handler.go (775)]: Libfuse::libfuse_read : error reading file 20u/hgt_20u_001_000.csv, handle: 128 [failed to download block]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1050)]: BlockCache::download : Failed to read 128=>20u/hgt_20u_001_000.csv from offset 0 [bad file descriptor]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [file_cache.go (1167)]: FileCache::ReadInBuffer : error [couldn't find fd in handle] 20u/hgt_20u_001_000.csv\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (1042)]: BlockCache::download : 3 attempts to download a block have failed 128=>20u/hgt_20u_001_000.csv (index 0, offset 0)\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (743)]: BlockCache::getBlock : Failed to download block 0 for 128=>20u/hgt_20u_001_000.csv (read offset 0)\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [block_cache.go (594)]: BlockCache::ReadInBuffer : Failed to get Block 128=>20u/hgt_20u_001_000.csv offset 0 [failed to download block]\nMar 13 15:52:40 nc2wrgConverter blobfuse2[4947]: [/home/azureuser/] LOG_ERR [libfuse_handler.go (775)]: Libfuse::libfuse_read : error reading file 20u/hgt_20u_001_000.csv, handle: 128 [failed to download block]\n\n\n```\n\nwhat could be the reason? \n\nI mount the blob container to a vm in azure and in the vm terminal i can read and write files. but existing files in the blob container can not be read."}, {"author": "syeleti-msft", "comment": "Hi @fenglinnorway,\n```\ncomponents:\n - libfuse\n - block_cache\n - file_cache\n - attr_cache\n - azstorage\n```\nas for data caching you can only use one caching mechanism (i.e., file_cache or block_cache) else the behavior is undefined.\nPlease remove block_cache/file_cache from the above the components section."}, {"author": "vibhansa-msft", "comment": "Closing this as there is no update. Kindly update your config and reopen the item if the issue persists."}]} +{"id": "issue-1639", "type": "issue", "title": "RAM leakage in blobfuse2 > 2.3.0", "body": "Hello,\nRelated to previously closed issue: https://github.com/Azure/azure-storage-fuse/issues/1617\n\nWe are using blobfuse2 on AKS via CSI blob driver, latest version.\n\nFew days ago we upgraded node pool to latest image and it automatically installed blobfuse version 2.4.0\n\nWhen we ran our performance tests, nodes started to transition to NotReady state after very short period of time.\n\nAfter conducting debug session, we realized `blobfuse2` is not freeing RAM at all. It just keep growing until host becomes unresponsive due to lack of memory.\n\nI conducted tests on following blobfuse2 versions: \n- 2.3.0 -> no issue\n- 2.3.2 -> issue persist\n- 2.4.0 -> issue persist\n- 2.4.1 -> issue persist\n\nWe are using Blob CSI via PV (RBAC UAMI auth) using below mount options:\n\n```yaml\nmountOptions:\n - '-o allow_other'\n - '--file-cache-timeout-in-seconds=0'\n - '-o attr_timeout=0'\n - '-o entry_timeout=0'\n - '-o negative_timeout=0'\n - '--attr-timeout=0'\n - '--entry-timeout=0''\n - '--cancel-list-on-mount-seconds=10'\n - '--block-cache\n```\n\nWe are using blob cache due to file cache limit (it doesn't clean up folders and inode limit is reached)\n\nWe tried `-o direct_io` but performance was very poor and not acceptable by our SLA\n\nAny suggestions are welcome, thanks!", "author": "Vegoo89", "created_at": "2025-02-19T14:35:06+00:00", "comments": [{"author": "vibhansa-msft", "comment": "How many files do you have in your container which shall list as part of this mount ?"}, {"author": "Vegoo89", "comment": "Hi,\nEach node has 3 blobfuse2 processes running that are mapped to different containers.\n\nPer one test (around 4000 tasks), we are producing ~10 small files per task with 800 tasks per minute ratio\n\nOne container is used for writing and reading of small files, mostly JSONs (reads are performed only after write is done, there is no integrity issue).\nSecond and third hold machine learning models (transformer models) and adapters. \nBasic model is loaded once and kept in memory of 3 microservices, while adapters - which are very small - are loaded and switched constantly."}, {"author": "vibhansa-msft", "comment": "As you are using block-cache and you have not set any upper limit on the cache usage, each blobfuse instance running will try to use 80% of the available memory by default. Dur to multiple instnaces your memory is running low. You need to set a max memory usage limit for blockc-cahe using `--block-cache-pool-size=` cli parameter."}, {"author": "Vegoo89", "comment": "I thought about it, but for version 2.3.0 limit was set to 4GB and process was using constantly 500-600 MB memory under load and it never reached this 4GB limit.\nNow memory keeps growing - for the same load - up to 20GB per process, so I can limit it to some value (no idea what would be correct one) but I doubt it is root of the problem."}, {"author": "vibhansa-msft", "comment": "With autoconfig we reserve 80% of memory space in single instance, assuming all resources are at our dispersal. If you are running multiple instances then you will need to manually restrict the memory usage. This might solve the problem. Try it out and let us know if that helps."}, {"author": "Vegoo89", "comment": "We are testing `--block-cache-pool-size=2000` and should have results today or tomorrow.\n\nHowever it is a bit concerning that this limit is out of scope on Kubernetes. \n\nHow do I make sure that my node doesn't die due to OOM when `blobfuse2` process is executed outside of the container, directly on the host so it doesn't respect `spec.resources.limits.memory` set by `blob-csi` daemonset?"}, {"author": "vibhansa-msft", "comment": "By default it asks OS about the total memory available on the system and then takes 80% of it. So if there are multiple instances are started in parallel then all may end up having the same value and thus they may end up over running the total limits. Restricting it manually might be the only option here I feel."}, {"author": "Vegoo89", "comment": "Ok I get it, question is why does it keeps allocating so much memory and doesn't release any in newest versions. \n\nIn version `2.3.0` memory usage - without setting `--block-cache-pool-size` peaked at 600MB. \n\nWith `2.3.2` and above it just keep growing.\n\nShould I expect significant performance issues if I set all PV to `--block-cache-pool-size=600` ?\n\nWas there some different memory management for block cache back in `2.3.0` that was removed or remade later on?"}, {"author": "vibhansa-msft", "comment": "In either version it shall not continuously rise. At certain point it shall stablize. Some of the memory related issues we have found recently are due to Go version upgrade and we are actively working on those."}, {"author": "syeleti-msft", "comment": "> In version 2.3.0 memory usage - without setting --block-cache-pool-size peaked at 600MB.\n\n> With 2.3.2 and above it just keep growing.\n\n1. Yes you are right, After 2.3.0 we made some changes on our memory management, when we reuse the buffer, in 2.3.0 we dont used to clear the buffer before using it(This might cause some data integrity issues) hence in >2.3.0 versions we try to clear the buffer by copying a zero buffer into the existing buffer. This is the reason why you see all the memory being used in the system. \n\n2. Currently in the latest release when you set the memory pool, the whole memory is used and only deallocated when the blobfuse terminates. This is a significant bottleneck and we are working on it to imporve the memory management in block cache.\n\nPlease refrain from using 2.3.0, there are many known data integrity issues, I suggest using the latest release."}, {"author": "Vegoo89", "comment": "> In either version it shall not continuously rise. At certain point it shall stablize. Some of the memory related issues we have found recently are due to Go version upgrade and we are actively working on those.\n\nIn our use case it doesn't stabilize for `>2.3.0`. It grows to ~20GB per process and causes node to become unresponsive so AKS moves it to NotReady state and removes it from the pool.\n\n> > In version 2.3.0 memory usage - without setting --block-cache-pool-size peaked at 600MB.\n> \n> > With 2.3.2 and above it just keep growing.\n> \n> 1. Yes you are right, After 2.3.0 we made some changes on our memory management, when we reuse the buffer, in 2.3.0 we dont used to clear the buffer before using it(This might cause some data integrity issues) hence in >2.3.0 versions we try to clear the buffer by copying a zero buffer into the existing buffer. This is the reason why you see all the memory being used in the system.\n> 2. Currently in the latest release when you set the memory pool, the whole memory is used and only deallocated when the blobfuse terminates. This is a significant bottleneck and we are working on it to imporve the memory management in block cache.\n> \n> Please refrain from using 2.3.0, there are many known data integrity issues, I suggest using the latest release.\n\nThanks for explanation, appreciate it. Just last one thing I would love to understand is how setting `--block-cache-pool-size=600` would affect performance.\n\nWe are working mostly with small files. Would setting this value to higher number be required if we would operate on blobs bigger than 600 MB? I am trying to understand what would work best for our use case and what impact this setting has on performance of the node (considering we have few `blobfuse2` processes running there)."}, {"author": "syeleti-msft", "comment": "> We are working mostly with small files. Would setting this value to higher number be required if we would operate on blobs bigger than 600 MB? \n\nNo, doing operations on bigger blobs dont need more memory size\n\nDo you see any perf difference when using 600M as memsize for your usecase in the latest release?"}, {"author": "Vegoo89", "comment": "We did few rounds of testing today. Each performance test takes around 15 minutes and produces around 120k files.\n\n2.3.0 -> no `--block-cache-pool-size` -> average CPU utilization 58% per node\n2.4.1 -> `--block-cache-pool-size=2000` -> average CPU utilization 50% per node\n2.4.1 -> `--block-cache-pool-size=600` -> average CPU utilization 54% per node\n\nWe will also perform more tests in upcoming days on bigger nodes, but results look good.\n\nA note from my observation for version `2.4.1` - after tests are finished each `blobfuse2` process stays at its limit and doesn't release memory. Also it allocates around ~23MB over the limit.\n\nHowever logic for default limit of 80% host memory seems a bit off. I am not Go expert, but I checked the code and I don't see any guard that prevents OOM on the system in situation similar to described above. \n\nIn containerized environment it most likely would happen on all nodes at the same time since traffic is distributed evenly in most commons scenarios and it blows up entire worker node pool. Cluster recovery in our case takes up to 1 hour."}, {"author": "syeleti-msft", "comment": "> However logic for default limit of 80% host memory seems a bit off. I am not Go expert, but I checked the code and I don't see any guard that prevents OOM on the system in situation similar to described above.\n\nConstraints were placed in the code taking the fact that only one blobfuse instance would run per VM but after looking into your scenario, I think we should reconsider some things. Thanks for pointing out."}, {"author": "Vegoo89", "comment": "Thanks! Would be great, since - best to my knowledge - it is considered best practice to split data between multiple containers instead of putting everything to single one and it means running more than one blobfuse process on the node."}, {"author": "vibhansa-msft", "comment": "If you are using multiple mounts on the same vm/node/machine, then you shall not rely on autoconfig and manually configure memory limits for each instance of blobfuse. Running blobfuse instance acts independently and hence it is not aware that any other instance of blobfuse mount exists in the system. As part of best practice, we will update our documents mentioning the same. Is there any other open query /question on this thread?"}, {"author": "Vegoo89", "comment": "Point of this issue was memory leak that started to happen after upgrade to newer blobfuse version"}, {"author": "vibhansa-msft", "comment": "In block-cahce mode there was a excessive memory usage problem, which has been fixed with version 2.5.0. Kindly upgrade to 2.5.0 and revalidate."}, {"author": "vibhansa-msft", "comment": "Closing this item. Kindly revalidate the scenario with 2.5.0 version and reopen if the issue persists."}]} +{"id": "issue-1632", "type": "issue", "title": "Clarification - temporary network interruption", "body": "Hi all,\n\nwe will have a short network maintenance which will disconnect the server running blobfuse from the internet for approx 15 minutes.\nWhat happens if a client tries to write a new file to the mounted folder in this time?\nWe use file caching.\n\nThanks in advance!", "author": "casimir-dus", "created_at": "2025-02-14T15:51:21+00:00", "comments": [{"author": "syeleti-msft", "comment": "Hi @casimir-dus, It depends on the configuration file for your blobfuse mount.\nIf network interruption happens **after** opening the file then all the writes will be success, but the call to the \"close\" operation would fail.\nIf network interruption happens **before** opening the file, then the status of the open file depends on the timeout you set for the following parameters in your config file.\n```\nlibfuse:\n attribute-expiration-sec: 300\n entry-expiration-sec: 300\n negative-entry-expiration-sec: 300\n\nattr_cache:\n timeout-sec: 300\n```\nIf the timeout has not expired then the open/create call to the file may still get success and subsequent writes would get success but eventually \"close\" operation would fail on the file.\nTo avoid such behaviors, avoid setting the timeout values to large and always check the status of the close()/fsync() operation in your user application.\n\nThanks!"}, {"author": "casimir-dus", "comment": "Hi @syeleti-msft,\nthanks!\nWhat would happen if a new file would be created?"}, {"author": "syeleti-msft", "comment": "when you try to create a new file blobfuse gets the metadata of the file from the azure storage, but there is a chance that this metadata was already cached if you do some operations on this file before the network gets interrupted and also your cache timeout is not expired. so create and following read/writes will give success but eventually close call will fail."}]} +{"id": "issue-1630", "type": "issue", "title": "Unable to use \"mount all\" when a container exists with a $ sign in the name (like \"$web\")", "body": "We have a storage account that has a container named \"$web\" within it.\n\nI believe this is very simply an issue where something is probably catching \"$web\" and interpreting it as a variable rather than being properly escaped.\n\nConfiguration\n\n```\nvi /etc/blobfuse2/config.yaml\n#\n# Refer ./setup/baseConfig.yaml for full set of config parameters\nlogging:\n type: syslog\n level: log_debug\n\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n\nfile_cache:\n path: /mnt/blobfuse2tmp\n timeout-sec: 120\n max-size-mb: 4096\n\nattr_cache:\n timeout-sec: 7200\n\nazstorage:\n type: block\n account-name: MYACCOUNT\n sas: MYSAS\n mode: sas\n# \n```\n\nTest commands\n```\nmkdir ./test\nblobfuse2 mount all ./test --read-only=true --config-file=/etc/blobfuse2/config.yaml\n```\n\nResult\n```\nMounting container : $web to path test/$web\nFailed to mount container $web : Error: invalid config file [open /root/.blobfuse2/config_.yaml: no such file or directory]\n```\n\nIf you look in /root/.blobfuse2, you'll see these files:\n```\ntotal 20\ndrwxr-xr-x 2 root root 4096 Feb 13 01:47 ./\ndrwx------ 18 root root 4096 Feb 13 01:44 ../\n-rw-r--r-- 1 root root 4 Feb 13 01:47 _home_admin_k_inventory.pid\n-rw-r--r-- 1 root root 654 Feb 13 01:47 'config_$web.yaml'\n-rw-r--r-- 1 root root 669 Feb 13 01:47 config_inventory.yaml\n```\n\nThe other container \"inventory\" mounts successfully.\n\nAs you can see above, blobfuse2 is complaining about \"/root/.blobfuse2/config_.yaml\" not existing, but there is a file called 'config_$web.yaml', so I bet $web is not being escaped in some internal command and results in a blank value being used.", "author": "angrygreenfrogs", "created_at": "2025-02-12T15:55:58+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "@angrygreenfrogs, in blobfuse mount command, the config file name can be provided as a variable, and blobfuse resolve that variable first before running the command.\nThis is what's happening in your case.\n$web is being considered a variable, which is being replaced by an empty value.\nAre there any specific use case for using special character in your container name?"}, {"author": "angrygreenfrogs", "comment": "Thanks for the reply!\n\nApologies, but I don't believe that's entirely accurate.... I'm with you that the problem in my case is that my \"$web\" container is being incorrectly interpreted as a variable (which is blank in the environment), but I still believe this is a bug.\n\nI've even found that I can work-around this issue in a hacky way by exporting a variable with that name, which tricks the executed shell command into working\n\n```\nweb='$web'\nexport web\nblobfuse2 mount all ./test --read-only=true --config-file=/etc/blobfuse2/config.yaml\n```\n\nI've never really done much with Go, but I suspect the problem is inside mountAllContainers(), which calls updateCliParams() to process CLI parameters that are called with exec.Command to execute the individual mount commands.\n\nI believe this is simply a lack of escaping the parameters correctly - effectively allowing a shell injection issue.\n\nIt's not uncommon to have container names like $web - that's the default container name used by Azure for serving static web content from a storage container (which is what our container is being used for).\n\nFor example: https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-static-website"}, {"author": "jainakanksha-msft", "comment": "This failure not only comes with mountall command.\nEven the command `blobfuse2 mount ./test --read-only=true --config-file=/home/abc/.blobfuse2/config_$web.yaml` also fails.\nReason being we resolve all the variables while evaluating the value of `config_file`."}, {"author": "vibhansa-msft", "comment": "in case of `mount all` you have an option to exclude certain containers from mounting, did you try using that ?"}, {"author": "vibhansa-msft", "comment": "Did above suggestion help you in resolving the issue ? If so, kindly close this item here. If there are no updates for next couple of days we will close this item from our end,"}, {"author": "angrygreenfrogs", "comment": "Hi @vibhansa-msft - Thanks for checking, however, no that doesn't actually solve the problem/bug.\n\nWe could exclude containers like $web from being mounted, but we actually want those containers to be mounted for our use case, so that work-around doesn't really help.\n\nAgain, this seems like a legitimate bug where any container with a dollar sign in the name won't be handled correctly.\n\nIf it can't be fixed, then we'll just do our best with other work-arounds for now."}, {"author": "vibhansa-msft", "comment": "I am able to mount \"$web\" independently (not using mount all). Let me validate if there is an issue specifically with `mount all` command or not."}, {"author": "vibhansa-msft", "comment": "I see the issue is only when the config file name includes \"$\" in the path and shell or blobfuse tries to resolve that assuming its an env variable."}]} +{"id": "issue-1625", "type": "issue", "title": "Unable to view files in mounted directory using blobfuse2", "body": "Despite following the correct steps for mounting an Azure Blob Storage container using blobfuse2, we are unable to see any files or directories in the mounted folder. The mount command completes without errors, and the directory is successfully mounted, but it remains empty.\n\n### Steps to Reproduce:\n\nInstall blobfuse2 version 2.4.0 on Ubuntu 24.04.\nRun blobfuse2 mount /home/user/drive --config-file=/home/user/.blobfuse2/config.yaml --allow-other.\nNavigate to /home/user/drive and attempt to view files using ls.\nObserve that no files or folders are displayed, even though the Azure Blob container contains files.\n\n### Expected Behavior:\nThe mounted directory should show the contents (files and folders) from the Azure Blob Storage container.\n\n### Actual Behavior:\nThe directory is empty, even though the mount process was completed successfully and there were no apparent errors during mounting.\n\n### Environment:\n\nOS: Ubuntu 24.04\nblobfuse2 version: 2.4.0\nAzure Blob Storage account\nAuthentication Mode: Managed Identity (MSI)\n\n###Additional Information:\n\nThe mount process completes without any errors.\nLogs show that the mount is initialised, and there are no errors related to the process.\nPermissions for the Managed Identity are correctly set with the Storage Blob Data Contributor role.\nThe config.yaml file is correctly configured with allow-other: true and correct storage account/container details.\n\n###Troubleshooting Steps Already Taken:\n\nVerified that the container contains files via Azure CLI.\nEnsured the correct permissions for the Managed Identity are set.\nUnmounted and remounted the filesystem multiple times.", "author": "Edil-Camil", "created_at": "2025-02-05T20:36:18+00:00", "comments": [{"author": "vibhansa-msft", "comment": "Kindly enable debug logs and share the log file so that we can investigate. After mounting your files might not be visible immediately as there is a cool down period to control the cost to customer. After 10 seconds or so you can try again and validate. If you want immediate results, then there is a knob to reduce this period to 0 as well."}]} +{"id": "issue-1624", "type": "issue", "title": "Is the following setup valid (sftp blob to fuse on linux)", "body": "I've got an azure blob sftp server. Files are uploaded there and a linux machine mounts this via fusefs. Sometime we notice that there are empty files. \n\nThe files are processed after the sftp sessions. So they are quite rapidly \"processed\" and it feels that the files on linux aren't yet there. \n\nIs this something I can fix on the fuse fs side without resorting to az copy? \n\n\nmount options config file\n\n```\n\nallow-other: true\nnonempty: true\n\nlogging:\n type: syslog\n level: log_warning\n\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n default-permission: 0666\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n\nfile_cache:\n path: /fusecache\n max-size-mb: 1024\n timeout-sec: 120\n\nblock_cache:\n block-size-mb: 32\n mem-size-mb: 4096\n prefetch: 80\n parallelism: 128\n\nattr_cache:\n no-symlinks: true\n timeout-sec: 7200\n\nazstorage:\n type: adls\n account-name: censored\n container: censored\n mode: msi\n objid: censored\n\n\n```\n\nfuse-libs-2.9.7-16.0.1.el8.x86_64\nfuse-common-3.3.0-16.0.1.el8.x86_64\nfuse3-libs-3.3.0-16.0.1.el8.x86_64\nblobfuse2-2.3.2-1.x86_64\nfuse3-3.3.0-16.0.1.el8.x86_64\n\nOracle enterprise linux 8.8", "author": "MalfuncEddie", "created_at": "2025-02-05T13:40:39+00:00", "comments": [{"author": "vibhansa-msft", "comment": "If you are looking for an instance sync with your storage account then you need to disable all kind of caching that blobfuse or your linux kernel is doing. To achieve so you can use \"-o direct_io\" option in your mount command."}, {"author": "MalfuncEddie", "comment": "Hi, \n\nI added the option in /etc/fstab and via a manual mount.\n\nis there a way I can check if it is active? The mount command\n\n```\nsudo blobfuse2 /mnt --config-file=/etc/fuse/sftp.yml --use-https=true -o allow_other -o direct_io\nmount | grep blob\nblobfuse2 on /mnt type fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other,max_read=1048576)\n```\n\ndoes not show the direct_io option"}, {"author": "vibhansa-msft", "comment": "You can add \"direct_io\" in this list ((rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other,max_read=1048576))"}, {"author": "MalfuncEddie", "comment": "Yes that is what I did\n\n```\n # sudo grep fuse /etc/fstab\nblobfuse2 /azstorage fuse3 defaults,direct_io,_netdev,--config-file=/etc/fuse/sftp.yml 0 0\n # mount | grep blob\nblobfuse2 on /azstorage type fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other,max_read=1048576,_netdev)\n```\n\n(did systemctl daemon-reload and a umount/ mount). \n\nIt's just strange that it is missing since I does feel like direct_io resolved it."}, {"author": "vibhansa-msft", "comment": "I still do not see direct_io in the fstab entry you have posted above. Also, if its difficult for you to provide additional options in fstab file, a simple workaround is to execute a shell script from fstab and inside shell script you write down the exact blobfuse command you wish to execute."}, {"author": "vibhansa-msft", "comment": "Kindly close this item if direct_io has resolved your issue."}, {"author": "MalfuncEddie", "comment": "> I still do not see direct_io in the fstab entry you have posted above.\n\n # sudo grep fuse /etc/fstab\nblobfuse2 /azstorage fuse3 defaults,**direct_io**,_netdev,--config-file=/etc/fuse/sftp.yml 0 0\n # mount | grep blob\nblobfuse2 on /azstorage type fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other,max_read=1048576,_netdev)\n\nbut I will close it"}]} +{"id": "issue-1617", "type": "issue", "title": "RAM lacking after upgrading blobfuse2 from blobfuse2-2.3.2 to blobfuse2-2.4.0", "body": "Hello,\nI was using blobfuse2-2.3.2 but with upgrade to blobfuse2-2.4.0 I'm experiencing RAM leaking after upgrade.\nOS: Red Hat Enterprise Linux 8.10\n\nI'm using blobfuse2 to push data from VMSS nodes into blob storage - every node is writing data into blob constantly - files are divided/distinguished by date and hours (every passing hour is next new file).\n\nAfter upgrading to newest version my memory looks like this:\n\n![Image](https://github.com/user-attachments/assets/d7fb3abd-2cd3-4016-922d-63e375376589)\nand eventually going to 0 with error:\n\n![Image](https://github.com/user-attachments/assets/0fa82253-5eb9-4f24-8b45-61ab7d8925b0)\nRAM on every machine is 8GM. I'm limiting memory usage for blobfuse2 up to 4GB leaving another 4GB for other apps and OS.\n\nI was using stream config before upgrade for more than year without issues, but I learned that stream mode was changed in newest version to block-cache, so I changed config to this:\n```\n\nlogging:\n type: syslog\n level: log_info\n\ncomponents:\n - libfuse\n - stream\n# - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n ignore-open-flags: true\n\n#stream:\nblock_cache:\n block-size-mb: 32\n mem-size-mb: 4096\n prefetch: 64\n parallelism: 16\n file-caching: false\n\n\n#attr_cache:\n# timeout-sec: 7200\n\nazstorage:\n type: block\n account-name:\n account-key:\n endpoint:\n mode: key\n container:\n```\n\nPlease help me understand if my config is correct and how to overcome this RAM issue.", "author": "Werpak", "created_at": "2025-01-24T09:30:33+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "In components block use this\n```\ncomponents:\n - libfuse\n - block_cache\n - azstorage\n```\n \nIn block_cache block use this\n\n```\nblock_cache:\n block-size-mb: 32\n mem-size-mb: 4096\n```\n\nWhat's the file size you are writing to storage?"}, {"author": "Werpak", "comment": "I'm writing to storage around 10GB per each hour during days and around 6gb during nights.\nEach passing hour has own unique file."}, {"author": "jainakanksha-msft", "comment": "When discussing a 10GB or 6GB file, what is your write throughput? \n\nKeep in mind that Blobfuse has only 4GB of memory available to process the content and as per your graph crash happens around 5GB. \nWith your machine size being 8GB, it indicates that the write throughput might be generating more data than the memory can handle.\nYou can try with higher memory size VM.\n Is your write operation exceeding this limit, causing Blobfuse to run out of memory or crash? \n\nTo verify, run your Blobfuse mount with the `--foreground` flag: \nExample: `./blobfuse2 mount --foreground`"}, {"author": "Werpak", "comment": "OK I'll verify throughput.\nI understand that I can try use bigger VM and also I know that 10GB file cannot fit into 4GB RAM limit.\nI just want to understand based on this change:\n![Image](https://github.com/user-attachments/assets/753aeaeb-1d66-47a2-9c5e-c59ed03afcef)\n\nPlease tell me if can I achieve behavior of blobfuse2 similar to stream option (before upgrade) where file was constantly updated with new data every passing few minutes, like append operation (now content of file is dump into storage when blobfuse2 is opening new file)."}, {"author": "Werpak", "comment": "Here are logs from start of blobfuse with --foreground flag:\n\nsudo journalctl -u blobfuse2.service | tail -100\n-- Logs begin at Sun 2025-01-26 11:13:37 UTC, end at Sun 2025-01-26 11:14:50 UTC. --\nJan 26 11:13:57 l000002 systemd[1]: Starting A virtual file system adapter for Azure Blob storage\nJan 26 11:13:59 l000002 blobfuse2[2098]: [] LOG_DEBUG [custom.go (54)]: initializePlugins: No plugins to load.\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.4.0 on [Red Hat Enterprise Linux 8.10 (Ootpa)]\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [mount.go (439)]: Mount Command: [/usr/bin/blobfuse2 mount /logstash/blobfuse2/prodcontent --config-file=/logstash/blobfuse2/config/StreamingConfig.yaml --log-level=LOG_TRACE --foreground]\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_TRACE\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse.go (284)]: Libfuse::Configure : libfuse\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse.go (252)]: Libfuse::Validate : UID 0, GID 0\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only false, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 240, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace false, extension , disable-writeback-cache false, dirPermission 511, mountPath /logstash/blobfuse2/prodcontent, umask 0\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_cache.go (208)]: BlockCache::Configure : block_cache\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [stream.go (78)]: Stream::Configure : stream\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [stream.go (109)]: Stream to Block Cache::Configure : Buffer size 32, Block size 32, Handle limit 64, FileCaching false, Read-only false, StreamCacheMb 0, MaxBlocksPerFile 0\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [block_cache.go (333)]: BlockCache::Configure : block size 33554432, mem size 2147483648, worker 6, prefetch 11, disk path , max size 0, disk timeout 120, prefetch-on-open false, maxDiskUsageHit false, noPrefetch false\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [azstorage.go (84)]: AzStorage::Configure : azstorage\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [config.go (292)]: ParseAndValidateConfig : Parsing config\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [config.go (407)]: ParseAndValidateConfig : using the following proxy address from the config file:\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [config.go (518)]: ParseAndReadDynamicConfig : Reparsing config\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [config.go (504)]: ParseAndValidateConfig : account xxxx, container xxxx, account-type BLOCK, auth KEY, prefix , endpoint https://xxxxxx.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [config.go (507)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [config.go (508)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_CRIT [config.go (511)]: ParseAndValidateConfig : Telemetry : , honour-ACL false, disable-symlink true\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_blob.go (163)]: BlockBlob::SetupPipeline : Setting up\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_blob.go (144)]: BlockBlob::createServiceClient : Getting service client\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_blob.go (226)]: BlockBlob::SetPrefixPath : path\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_blob.go (180)]: BlockBlob::TestPipeline : Validating\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [mount.go (464)]: mount: Mounting blobfuse2 on /logstash/blobfuse2/prodcontent\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [azstorage.go (171)]: AzStorage::Start : Starting component azstorage\nJan 26 11:13:59 l000002 blobfuse2[2098]: fuse: warning: library too old, some operations may not not work\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_cache.go (136)]: BlockCache::Start : Starting component block_cache\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse.go (153)]: Libfuse::Start : Starting component libfuse\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse_handler.go (120)]: Libfuse::initFuse : Initializing FUSE3\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse_handler.go (163)]: Libfuse::initFuse : Registering fuse callbacks\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse_handler.go (167)]: Libfuse::initFuse : Populating fuse arguments\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse_handler.go (191)]: Libfuse::populateFuseArgs\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (179)]: Libfuse::initFuse : Mounting with fuse3 library\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [libfuse_handler.go (259)]: Libfuse::libfuse_init : init (read : 0, write 1048576, read-ahead 131072)\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (261)]: Libfuse::NotifyMountToParent : Notifying parent for successful mount\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (268)]: Libfuse::libfuse_init : Kernel Caps : 77594587\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (275)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_PARALLEL_DIROPS\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (281)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_AUTO_INVAL_DATA\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (288)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_READDIRPLUS\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (294)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_ASYNC_READ\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (300)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_SPLICE_WRITE\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (312)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_WRITEBACK_CACHE\nJan 26 11:13:59 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_INFO [libfuse_handler.go (330)]: Libfuse::libfuse_init : Ignoring max_write for fuse minor 3\nJan 26 11:14:45 l000002 blobfuse2[2098]: [/logstash/blobfuse2/prodcontent] LOG_TRACE [block_blob.go (515)]: BlockBlob::GetAttr : name Fortinet/y=2025/m=01/d=26/h=11/m=00/xxxx.json"}, {"author": "vibhansa-msft", "comment": "With block-cache did it solve the increasing memory usage issue for you?\nFor the constantly changing file, if your application closes the handle and opens it again every time you want to process it and needs the latest contents then you need to use \"-o direct_io\" option this will disable all sort of caching so that you always get the latest contents. As caching is disabled it will have additional cost and delays (as every time we need to check stuff with backend storage)."}, {"author": "vibhansa-msft", "comment": "Downgrade of go version has been merged in feature branch. Out next release shall have the fix. Feel free to reopen this item or create a new one if the issue persist after the 2.4.1 release."}]} +{"id": "issue-1615", "type": "issue", "title": "Cant mount some folders, 2 out of 10", "body": "Which version of blobfuse was used?\n2.4.0\n\nWhich OS distribution and version are you using?\nUbuntu 22.04\n\nconfig-file\nallow-other: true\n\nlogging:\ntype: syslog\n\ncomponents:\n- libfuse\n- stream\n- attr_cache\n- azstorage\n\n\nlibfuse:\ndefault-permission: 0777\n\nstream:\nblock-size-mb: 0\nblocks-per-file: 0\ncache-size-mb: 0\n\nattr_cache:\ntimeout-sec: 120\n\nazstorage:\ntype: block\naccount-name: hodukwprodbitwdnsa\nendpoint: https://hodukwprodbitwdnsa.blob.core.windows.net/\nmode: key\naccount-key: **********\nvirtual-directory: false\nhonour-acl: false\n\nmountall:\ncontainer-allowlist:\n- ca-certificates\n- core\n- docker\n- env\n- identity\n- letsencrypt\n- nginx\n- scripts\n- ssl\n- web\n\n**What was the issue encountered? **\nI am using blobfuse to mount 10 folders, for some reason now, two folders will not mount. This all worked flawlessly for around a year. When trying to mount the folder, i get the error message \"Error: failed to initialize new pipeline [config error in file_cache error [tmp-path not set]]\". For context, I use blobfuse to share folders between two machines setup to self host Bitwarden, the other machine is not experiencing this issue and has the same mirrored setup.\n\nHave you found a mitigation/solution?\nNo\n\nPlease share logs if available\n\npulled using grep blobfuse /var/log/syslog\n\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888052] [ 882] 1001 882 1214161 147054 146574 480 0 1572864 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888054] [ 927] 1001 927 1099349 10815 10335 480 0 376832 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888058] [ 1002] 1001 1002 1087125 2626 2146 480 0 270336 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888060] [ 1035] 1001 1035 1105787 7214 6734 480 0 360448 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888062] [ 1061] 1001 1061 1143169 28185 27737 448 0 552960 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888064] [ 1102] 1001 1102 1116286 3132 2652 480 0 311296 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888065] [ 1135] 1001 1135 1164132 85190 84710 480 0 1036288 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888067] [ 1332] 1001 1332 1073019 14919 14471 448 0 425984 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888071] [ 1445] 1001 1445 1050296 6732 6252 480 0 335872 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888144] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=docker-9ec7de136e7401d0b22d4b344233147eec7a7b024bc6555311fbe5f011d4b5a5.scope,mems_allowed=0,global_oom,task_memcg=/system.slice/cron.service,task=blobfuse2,pid=882,uid=1001\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888188] Out of memory: Killed process 882 (blobfuse2) total-vm:4856644kB, anon-rss:586296kB, file-rss:1920kB, shmem-rss:0kB, UID:1001 pgtables:1536kB oom_score_adj:0\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 927 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1002 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1035 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1061 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1102 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1135 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1332 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1445 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 927 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1002 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1035 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1061 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1102 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1135 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1332 (blobfuse2) in control group while starting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1445 (blobfuse2) in control group while starting unit. Ignoring.", "author": "Jamie-Hodge", "created_at": "2025-01-21T08:07:40+00:00", "comments": [{"author": "Jamie-Hodge", "comment": "I thought this issue was resolved after some general updates and a reboot, but the issue persists if anyone has any suggestions?"}, {"author": "ashruti-msft", "comment": "Hi @Jamie-Hodge, the error `Error: failed to initialize new pipeline [config error in file_cache error [tmp-path not set]]` is only shown if you are using file-cache mode in blobfuse and have not mentioned a temp directory path in the config file. I see you are using stream mode in your config, so this error doesn't seem correct. Can you confirm if this is the error from the current mount using the above config file?\n\nAlso, we are planning to deprecate streaming mode in blobfuse2 so I would advise to move to file_cache mode or block_cache mode depending on your requirements."}, {"author": "Jamie-Hodge", "comment": "I can confirm this is the error i receive. When i try to update bitwarden, it updates some files in the 10 mounted folders, the env folder being one of them. However, it throws an error that the endpoint \"env\" is not connected. I also cannot browse its contents, but i can with the other folders. If you unmount the folder, or unmount all folders and try remounting, you get the above error relating to the env folder only. \n\nI will look into cache modes."}, {"author": "ashruti-msft", "comment": "Can you share debug logs for your run? Thanks!"}, {"author": "Jamie-Hodge", "comment": "Not sure where to find them if you can offer some advice? Thanks."}, {"author": "Jamie-Hodge", "comment": "These are from syslog, we only have basic logging enabled, let me know if i need to add the debug level logging to our config file.\n\n Jan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888052] [ 882] 1001 882 1214161 147054 146574 480 0 1 572864 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888054] [ 927] 1001 927 1099349 10815 10335 480 0 376832 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888058] [ 1002] 1001 1002 1087125 2626 2146 480 0 270336 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888060] [ 1035] 1001 1035 1105787 7214 6734 480 0 360448 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888062] [ 1061] 1001 1061 1143169 28185 27737 448 0 552960 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888064] [ 1102] 1001 1102 1116286 3132 2652 480 0 311296 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888065] [ 1135] 1001 1135 1164132 85190 84710 480 0 1 036288 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888067] [ 1332] 1001 1332 1073019 14919 14471 448 0 425984 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888071] [ 1445] 1001 1445 1050296 6732 6252 480 0 335872 0 0 blobfuse2\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888144] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=docker-9ec7 de136e7401d0b22d4b344233147eec7a7b024bc6555311fbe5f011d4b5a5.scope,mems_allowed=0,global_oom,task_memcg=/system.slice/cron.s ervice,task=blobfuse2,pid=882,uid=1001\nJan 21 06:45:38 JHBAZUBITWDN02 kernel: [34039.888188] Out of memory: Killed process 882 (blobfuse2) total-vm:4856644kB, anon -rss:586296kB, file-rss:1920kB, shmem-rss:0kB, UID:1001 pgtables:1536kB oom_score_adj:0\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 927 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1002 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1035 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1061 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1102 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1135 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1332 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1445 (blobfuse2) remains running after unit stopped.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 927 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1002 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1035 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1061 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1102 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1135 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1332 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 06:45:40 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1445 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 09:03:07 JHBAZUBITWDN02 CRON[629]: (bitwarden) CMD (blobfuse2 mount all /opt/bitwarden/bwdata --config-file=/opt/bitw arden/bwdata/bf-bwdata-ak.yaml)\nJan 21 22:52:24 JHBAZUBITWDN02 CRON[683]: (bitwarden) CMD (blobfuse2 mount all /opt/bitwarden/bwdata --config-file=/opt/bitw arden/bwdata/bf-bwdata-ak.yaml)\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823631] [ 877] 1001 877 1136506 10785 10337 448 0 368640 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823634] [ 896] 1001 896 1132474 2600 2120 480 0 286720 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823637] [ 918] 1001 918 1146776 6632 6152 480 0 364544 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823643] [ 943] 1001 943 1142973 2504 2024 480 0 282624 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823646] [ 982] 1001 982 1225677 177992 177512 480 0 1 896448 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823649] [ 1003] 1001 1003 1190532 23086 22606 480 0 528384 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823652] [ 1023] 1001 1023 1130328 14775 14327 448 0 417792 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823658] [ 1083] 1001 1083 1130363 6610 6130 480 0 356352 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823661] [ 1109] 1001 1109 1199242 84594 84114 480 0 1 048576 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823667] [ 1146] 1001 1146 1205905 137628 137148 480 0 1 503232 0 0 blobfuse2\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823800] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=mde,mems_al lowed=0,global_oom,task_memcg=/system.slice/cron.service,task=blobfuse2,pid=982,uid=1001\nJan 21 22:56:12 JHBAZUBITWDN02 kernel: [ 248.823850] Out of memory: Killed process 982 (blobfuse2) total-vm:4902708kB, anon -rss:710048kB, file-rss:1920kB, shmem-rss:0kB, UID:1001 pgtables:1852kB oom_score_adj:0\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 877 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 896 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 918 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 943 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1003 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1023 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1083 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1109 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1146 (blobfuse2) remains running after unit stopped.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 877 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 896 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 918 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 943 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1003 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1023 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1083 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1109 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 22:56:14 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1146 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 22:59:53 JHBAZUBITWDN02 CRON[638]: (bitwarden) CMD (blobfuse2 mount all /opt/bitwarden/bwdata --config-file=/opt/bitw arden/bwdata/bf-bwdata-ak.yaml)\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340126] [ 873] 1001 873 1134523 2621 2141 480 0 294912 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340129] [ 903] 1001 903 1192804 85109 84629 480 0 1 044480 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340132] [ 966] 1001 966 1109625 2595 2115 480 0 286720 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340139] [ 1026] 1001 1026 1083064 6587 6075 512 0 335872 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340143] [ 1052] 1001 1052 1157729 23067 22587 480 0 536576 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340145] [ 1093] 1001 1093 1183376 137629 137149 480 0 1 490944 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340148] [ 1115] 1001 1115 1186508 199127 198647 480 0 2 052096 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340152] [ 1159] 1001 1159 1089467 10717 10237 480 0 389120 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340158] [ 1222] 1001 1222 1062520 14799 14287 512 0 401408 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340161] [ 1287] 1001 1287 1046136 6761 6249 512 0 344064 0 0 blobfuse2\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340290] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=docker-9681 d43926c082fcc091564d8d014f394992289527b101803c05e15d4beefdb3.scope,mems_allowed=0,global_oom,task_memcg=/system.slice/cron.s ervice,task=blobfuse2,pid=1115,uid=1001\nJan 21 23:01:55 JHBAZUBITWDN02 kernel: [ 130.340340] Out of memory: Killed process 1115 (blobfuse2) total-vm:4746032kB, ano n-rss:794588kB, file-rss:1920kB, shmem-rss:0kB, UID:1001 pgtables:2004kB oom_score_adj:0\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 873 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 903 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 966 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1026 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1052 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1093 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1159 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1222 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Unit process 1287 (blobfuse2) remains running after unit stopped.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 873 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 903 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 966 (blobfuse2) in control group while star ting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1026 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1052 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1093 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1159 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1222 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:01:57 JHBAZUBITWDN02 systemd[1]: cron.service: Found left-over process 1287 (blobfuse2) in control group while sta rting unit. Ignoring.\nJan 21 23:12:31 JHBAZUBITWDN02 CRON[667]: (bitwarden) CMD (blobfuse2 mount all /opt/bitwarden/bwdata --config-file=/opt/bitw arden/bwdata/bf-bwdata-ak.yaml)"}, {"author": "ashruti-msft", "comment": "Add \n`logging:`\n `type: base`\n `level: log_debug` \n `file-path: ./blobfuse2.log`\nin your config file and share the log file with me. Thanks!"}, {"author": "vibhansa-msft", "comment": "Closing as there is no update. Feel free to reopen once you have required info."}]} +{"id": "issue-1612", "type": "issue", "title": "File open error for storage mounted with subdirectory and block cache.", "body": "When mounting a storage subdirectory with a block cache, opening of files may fail. Using a file cache or without a subdirectory shows no such problems.\n\n```bash\n$ sha256sum somefile.bin\nLOG_TRACE [libfuse_handler.go (687)]: Libfuse::libfuse_open : somefile.bin\nLOG_TRACE [block_cache.go (385)]: BlockCache::OpenFile : name=somefile.bin, flags=32768, mode=-rwxrwxrwx\nLOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : somefile.bin\nLOG_DEBUG [attr_cache.go (482)]: AttrCache::GetAttr : somefile.bin served from cache\nLOG_ERR [block_cache.go (389)]: BlockCache::OpenFile : Failed to get attr of somefile.bin [no such file or directory]\nLOG_ERR [libfuse_handler.go (721)]: Libfuse::libfuse_open : Failed to open somefile.bin [no such file or directory]\nsha256sum: somefile.bin: No such file or directory\n```\n\nInside the directory, some files can be opened and others not, seemingly arbitrarily. The success and failures are consistent across remounts though. The storage has a flat namespace and block blobs.\n\nFrom what I can discern, the failing code path is like this:\n1. [attr_cache.go:GetAttr returns an ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/attr_cache/attr_cache.go#L484).\n2. [block_cache.go:OpenFile picks up the error and returns the same ENOENT error](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/block_cache/block_cache.go#L390).\n3. [libfuse_handler.go:libfuse_open picks up the error and fails](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/libfuse/libfuse_handler.go#L723).\n\nconfig.yaml:\n```yaml\nallow-other: true\n\nlogging:\n type: syslog\n level: log_debug\n\ncomponents:\n - libfuse\n - block_cache\n - attr_cache\n - azstorage\n\nlibfuse:\n attribute-expiration-sec: 120\n entry-expiration-sec: 120\n negative-entry-expiration-sec: 240\n fuse-trace: true\n\nblock_cache:\n path: /tmp/blobfuse/block_cache\n block-size-mb: 32\n mem-size-mb: 8192\n prefetch: 20\n parallelism: 32\n\nfile_cache:\n path: /tmp/blobfuse/file_cache\n\nattr_cache:\n timeout-sec: 7200\n\nazstorage:\n type: block\n```\n\nInvoking blobfuse2 with `--subdirectory=somefolder` then may exhibit this failure. Not passing the subdirectory option or using a file cache seems to work well.\n\nThis was all done in wsl2.", "author": "listcrawler", "created_at": "2025-01-17T13:04:16+00:00", "comments": [{"author": "jainakanksha-msft", "comment": "The file \"somefile.bin\" which you are invoking is inside the \"somefolder\" or outside.\nIf it is outside, the call will fail because you have mounted over \"somefolder\".\nAnd you are trying to invoke an external file\nIf problem still persist share the logs and the snapshot of your storage structure."}, {"author": "listcrawler", "comment": "This is all inside a corporate tenant, so I can't share any of the specific details here.\n\nStorage file structure [updated with subdirectory]:\n```bash\nsome_storage_account/\n\u2514\u2500\u2500 some_container\n \u2514\u2500\u2500 some_subdirectory\n \u251c\u2500\u2500 some_file.bin # file open fails\n \u2514\u2500\u2500 some_file2.bin # file open succeeds\nanother_storage_account/\n\u2514\u2500\u2500 another_container\n \u2514\u2500\u2500 another_subdirectory\n \u251c\u2500\u2500 another_file.bin # file open succeeds\n \u2514\u2500\u2500 another_file2.bin # file open fail\n````\nAgain:\n* When using the `blobfuse2 --subdirectory some_subdirectory` option and `block_cache` configuration at the same time to mount a subdirectory in a container in a storage account:\n * `some_file.bin` open fails while `some_file2.bin` succeeds, and this is consistently so across remounts.\n * Changing storage account and container still shows the same problem.\n * What files fail to open seem to be random for different accounts and containers, but for a specific account and container, the behavior is consistent.\n* In other cases (i.e. mounting not having set both `--subdirectory` and `block_cache` at the same time) opening of files work without error.\n* The platform is wsl2.\n* You should have what I hope is the essential part of the log in the original description."}, {"author": "jainakanksha-msft", "comment": "Shared logs are incomplete. \nCould you please provide the /var/log/syslog for a comprehensive understanding of the issue?"}, {"author": "listcrawler", "comment": "Pasting redacted log below, please look for `myfileopenfail` and `myfileopensuccess`.\n\n```\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.4.0 on [Ubuntu 20.04.5 LTS]\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_INFO [mount.go (439)]: Mount Command: [blobfuse2 /tmp/blobfuse/mount --config-file=config.yaml --default-working-dir=/tmp/blobfuse/work_dir --tmp-path=/tmp/blobfuse/tmp --virtual-directory=true --read-only=true --container-name=mycontainer --subdirectory=mycontainer]\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_DEBUG\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [mount.go (441)]: Mount allowed on nonempty path : false\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse.go (284)]: Libfuse::Configure : libfuse\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_INFO [libfuse.go (252)]: Libfuse::Validate : UID 0, GID 0\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only true, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 240, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace true, extension , disable-writeback-cache false, dirPermission 511, mountPath /tmp/blobfuse/mount, umask 0\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (208)]: BlockCache::Configure : block_cache\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [block_cache.go (333)]: BlockCache::Configure : block size 33554432, mem size 8589934592, worker 32, prefetch 20, disk path /tmp/blobfuse/block_cache, max size 811569768038, disk timeout 120, prefetch-on-open false, maxDiskUsageHit false, noPrefetch false\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (136)]: AttrCache::Configure : attr_cache\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [attr_cache.go (162)]: AttrCache::Configure : cache-timeout 7200, symlink false, max-files 5000000\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [azstorage.go (84)]: AzStorage::Configure : azstorage\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [config.go (292)]: ParseAndValidateConfig : Parsing config\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_WARNING [config.go (363)]: ParseAndValidateConfig : account endpoint not provided, assuming the default .core.windows.net style endpoint\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_INFO [config.go (407)]: ParseAndValidateConfig : using the following proxy address from the config file: \nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [config.go (518)]: ParseAndReadDynamicConfig : Reparsing config\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [config.go (414)]: ParseAndValidateConfig : Getting auth type\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [config.go (419)]: ParseAndValidateConfig : Auth type spn\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (504)]: ParseAndValidateConfig : account mystorageaccount, container mycontainer, account-type BLOCK, auth SPN, prefix mycontainer, endpoint https://mystorageaccount.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (507)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (508)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (511)]: ParseAndValidateConfig : Telemetry : , honour-ACL false, disable-symlink true\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (163)]: BlockBlob::SetupPipeline : Setting up\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (144)]: BlockBlob::createServiceClient : Getting service client\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [azauth.go (82)]: azAuth::getAzAuth : Account: mystorageaccount, AccountType: BLOCK, Protocol: https, Endpoint: https://mystorageaccount.blob.core.windows.net/\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [azauthspn.go (73)]: AzAuthSPN::getTokenCredential : Using client secret for fetching token\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (226)]: BlockBlob::SetPrefixPath : path mycontainer\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (180)]: BlockBlob::TestPipeline : Validating\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=REDACTED&delimiter=REDACTED&maxresults=REDACTED&prefix=REDACTED&restype=REDACTED\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=REDACTED\nMon Jan 20 09:44:08 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https%3A%2F%2Flogin.microsoftonline.com%2Fmytenant%2Foauth2%2Fv2.0%2Fauthorize\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:44:10 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/1.998160913s, OpTime=1.998179497s) -- RESPONSE RECEIVED\n GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https%3A%2F%2Flogin.microsoftonline.com%2Fmytenant%2Foauth2%2Fv2.0%2Fauthorize\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Access-Control-Allow-Methods: REDACTED\n Access-Control-Allow-Origin: REDACTED\n Cache-Control: max-age=86400, private\n Client-Request-Id: REDACTED\n Content-Length: 980\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:44:10 GMT\n P3p: REDACTED\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: 5ec7a18e-4d0e-4e13-a2ab-5a07d1542a00\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:44:10 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:44:10 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:44:10 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\nMon Jan 20 09:44:10 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:44:11 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/508.710244ms, OpTime=508.741293ms) -- RESPONSE RECEIVED\n GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Access-Control-Allow-Methods: REDACTED\n Access-Control-Allow-Origin: REDACTED\n Cache-Control: max-age=86400, private\n Client-Request-Id: REDACTED\n Content-Length: 1753\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:44:10 GMT\n P3p: REDACTED\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: 37b15b17-e7b0-48d0-a395-e6a230f55d00\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:44:11 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:44:11 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:44:11 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\nMon Jan 20 09:44:11 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Content-Length: 302\n Content-Type: application/x-www-form-urlencoded; charset=utf-8\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:44:12 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/576.93395ms, OpTime=576.987589ms) -- RESPONSE RECEIVED\n POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Content-Length: 302\n Content-Type: application/x-www-form-urlencoded; charset=utf-8\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Cache-Control: no-store, no-cache\n Client-Request-Id: REDACTED\n Content-Length: 1412\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:44:11 GMT\n Expires: -1\n P3p: REDACTED\n Pragma: no-cache\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Clitelem: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: b9e23995-4a26-4993-9d09-3b88b94e0b00\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:44:12 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:44:12 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:44:12 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Authentication) : ClientSecretCredential.GetToken() acquired a token for scope \"https://storage.azure.com/.default\"\nMon Jan 20 09:44:12 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&maxresults=2&prefix=mycontainer&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n\nMon Jan 20 09:44:13 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/1.278728718s, OpTime=1.278799368s) -- RESPONSE RECEIVED\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&maxresults=2&prefix=mycontainer&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Content-Type: application/xml\n Date: Mon, 20 Jan 2025 09:44:12 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n Vary: REDACTED\n X-Ms-Request-Id: 84f840f1-a01e-00a3-5e1f-6b8f6b000000\n X-Ms-Version: 2024-05-04\n\nMon Jan 20 09:44:13 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:44:13 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:44:13 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_INFO [mount.go (464)]: mount: Mounting blobfuse2 on /tmp/blobfuse/mount\nMon Jan 20 09:44:13 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_DEBUG [mount.go (502)]: mount: foreground disabled, child = false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [mount.go (438)]: Starting Blobfuse2 Mount : 2.4.0 on [Ubuntu 20.04.5 LTS]\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [mount.go (439)]: Mount Command: [blobfuse2 /tmp/blobfuse/mount --config-file=config.yaml --default-working-dir=/tmp/blobfuse/work_dir --tmp-path=/tmp/blobfuse/tmp --virtual-directory=true --read-only=true --container-name=mycontainer --subdirectory=mycontainer]\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [mount.go (440)]: Logging level set to : LOG_DEBUG\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [mount.go (441)]: Mount allowed on nonempty path : false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse.go (284)]: Libfuse::Configure : libfuse\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse.go (252)]: Libfuse::Validate : UID 0, GID 0\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [libfuse.go (335)]: Libfuse::Configure : read-only true, allow-other true, allow-root false, default-perm 511, entry-timeout 120, attr-time 120, negative-timeout 240, ignore-open-flags true, nonempty false, direct_io false, max-fuse-threads 128, fuse-trace true, extension , disable-writeback-cache false, dirPermission 511, mountPath /tmp/blobfuse/mount, umask 0\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (208)]: BlockCache::Configure : block_cache\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [block_cache.go (333)]: BlockCache::Configure : block size 33554432, mem size 8589934592, worker 32, prefetch 20, disk path /tmp/blobfuse/block_cache, max size 811569754931, disk timeout 120, prefetch-on-open false, maxDiskUsageHit false, noPrefetch false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (136)]: AttrCache::Configure : attr_cache\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [attr_cache.go (162)]: AttrCache::Configure : cache-timeout 7200, symlink false, max-files 5000000\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [azstorage.go (84)]: AzStorage::Configure : azstorage\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [config.go (292)]: ParseAndValidateConfig : Parsing config\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_WARNING [config.go (363)]: ParseAndValidateConfig : account endpoint not provided, assuming the default .core.windows.net style endpoint\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [config.go (407)]: ParseAndValidateConfig : using the following proxy address from the config file: \nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [config.go (518)]: ParseAndReadDynamicConfig : Reparsing config\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [config.go (414)]: ParseAndValidateConfig : Getting auth type\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [config.go (419)]: ParseAndValidateConfig : Auth type spn\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (504)]: ParseAndValidateConfig : account mystorageaccount, container mycontainer, account-type BLOCK, auth SPN, prefix mycontainer, endpoint https://mystorageaccount.blob.core.windows.net/, MD5 false false, virtual-directory true, disable-compression false, CPK false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (507)]: ParseAndValidateConfig : use-HTTP false, block-size 0, max-concurrency 32, default-tier %!s(*generated.AccessTier=), fail-unsupported-op true, mount-all-containers false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (508)]: ParseAndValidateConfig : Retry Config: retry-count 5, max-timeout 900, backoff-time 4, max-delay 60, preserve-acl: false\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_CRIT [config.go (511)]: ParseAndValidateConfig : Telemetry : , honour-ACL false, disable-symlink true\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (163)]: BlockBlob::SetupPipeline : Setting up\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (144)]: BlockBlob::createServiceClient : Getting service client\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [azauth.go (82)]: azAuth::getAzAuth : Account: mystorageaccount, AccountType: BLOCK, Protocol: https, Endpoint: https://mystorageaccount.blob.core.windows.net/\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [azauthspn.go (73)]: AzAuthSPN::getTokenCredential : Using client secret for fetching token\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (226)]: BlockBlob::SetPrefixPath : path mycontainer\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [mount.go (464)]: mount: Mounting blobfuse2 on /tmp/blobfuse/mount\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [mount.go (502)]: mount: foreground disabled, child = true\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [mount.go (593)]: Mount::runPipeline : blobfuse2 pid = 1462, transfer pipe = /tmp/transferPipe_1462, polling pipe = /tmp/pollPipe_1462\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [azstorage.go (171)]: AzStorage::Start : Starting component azstorage\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (106)]: AttrCache::Start : Starting component attr_cache\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (136)]: BlockCache::Start : Starting component block_cache\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse.go (153)]: Libfuse::Start : Starting component libfuse\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (120)]: Libfuse::initFuse : Initializing FUSE3\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (163)]: Libfuse::initFuse : Registering fuse callbacks\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (167)]: Libfuse::initFuse : Populating fuse arguments\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (191)]: Libfuse::populateFuseArgs\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : blobfuse2\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : /tmp/blobfuse/mount\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : -o\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : entry_timeout=120,attr_timeout=120,negative_timeout=240,allow_other,ro,max_read=1048576,kernel_cache\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : -f\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : -ofsname=blobfuse2\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [libfuse_handler.go (239)]: Libfuse::populateFuseArgs : opts : -d\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (179)]: Libfuse::initFuse : Mounting with fuse3 library\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (259)]: Libfuse::libfuse_init : init (read : 0, write 1048576, read-ahead 131072)\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (261)]: Libfuse::NotifyMountToParent : Notifying parent for successful mount\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (268)]: Libfuse::libfuse_init : Kernel Caps : 52428763\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (275)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_PARALLEL_DIROPS\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (281)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_AUTO_INVAL_DATA\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (288)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_READDIRPLUS\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (294)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_ASYNC_READ\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (300)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_SPLICE_WRITE\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (312)]: Libfuse::libfuse_init : Enable Capability : FUSE_CAP_WRITEBACK_CACHE\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1452] : [/tmp/blobfuse/mount] LOG_INFO [mount.go (516)]: mount: Child [1462] mounted successfully at /tmp/blobfuse/mount\nMon Jan 20 09:44:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [libfuse_handler.go (327)]: Libfuse::libfuse_init : Setting 1MB max_write for fuse minor 9\nMon Jan 20 09:45:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [block_cache.go (1643)]: BlockCache::checkDiskUsage : current disk usage : 0.003906MB 0%\nMon Jan 20 09:45:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [block_cache.go (1644)]: BlockCache::checkDiskUsage : current cache usage : 0%\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : myfileopenfailure.bin\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (515)]: BlockBlob::GetAttr : name myfileopenfailure.bin\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (466)]: BlockBlob::getAttrUsingList : name myfileopenfailure.bin\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (529)]: BlockBlob::List : prefix myfileopenfailure.bin, marker \nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=REDACTED&delimiter=REDACTED&include=REDACTED&maxresults=REDACTED&prefix=REDACTED&restype=REDACTED\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=REDACTED\nMon Jan 20 09:45:55 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https%3A%2F%2Flogin.microsoftonline.com%2Fmytenant%2Foauth2%2Fv2.0%2Fauthorize\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/1.550193766s, OpTime=1.550423759s) -- RESPONSE RECEIVED\n GET https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https%3A%2F%2Flogin.microsoftonline.com%2Fmytenant%2Foauth2%2Fv2.0%2Fauthorize\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Access-Control-Allow-Methods: REDACTED\n Access-Control-Allow-Origin: REDACTED\n Cache-Control: max-age=86400, private\n Client-Request-Id: REDACTED\n Content-Length: 980\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:45:56 GMT\n P3p: REDACTED\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: 261a6e56-ee34-4d6e-999b-b87f5e053100\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/399.242185ms, OpTime=399.251827ms) -- RESPONSE RECEIVED\n GET https://login.microsoftonline.com/mytenant/v2.0/.well-known/openid-configuration\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Access-Control-Allow-Methods: REDACTED\n Access-Control-Allow-Origin: REDACTED\n Cache-Control: max-age=86400, private\n Client-Request-Id: REDACTED\n Content-Length: 1753\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:45:57 GMT\n P3p: REDACTED\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: 8e2f5a31-7391-454f-82b6-5370ff595800\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\nMon Jan 20 09:45:57 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Content-Length: 302\n Content-Type: application/x-www-form-urlencoded; charset=utf-8\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/483.761731ms, OpTime=483.802759ms) -- RESPONSE RECEIVED\n POST https://login.microsoftonline.com/mytenant/oauth2/v2.0/token\n Accept-Encoding: REDACTED\n Client-Request-Id: REDACTED\n Content-Length: 302\n Content-Type: application/x-www-form-urlencoded; charset=utf-8\n Return-Client-Request-Id: REDACTED\n User-Agent: azsdk-go-azidentity/v1.8.0 (go1.23.1; linux)\n X-Client-Cpu: REDACTED\n X-Client-Os: REDACTED\n X-Client-Sku: REDACTED\n X-Client-Ver: REDACTED\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Cache-Control: no-store, no-cache\n Client-Request-Id: REDACTED\n Content-Length: 1417\n Content-Security-Policy-Report-Only: REDACTED\n Content-Type: application/json; charset=utf-8\n Date: Mon, 20 Jan 2025 09:45:57 GMT\n Expires: -1\n P3p: REDACTED\n Pragma: no-cache\n Set-Cookie: REDACTED\n Strict-Transport-Security: REDACTED\n X-Content-Type-Options: REDACTED\n X-Ms-Clitelem: REDACTED\n X-Ms-Ests-Server: REDACTED\n X-Ms-Request-Id: 8e2f5a31-7391-454f-82b6-5370085a5800\n X-Ms-Srs: REDACTED\n X-Xss-Protection: REDACTED\n\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Authentication) : ClientSecretCredential.GetToken() acquired a token for scope \"https://storage.azure.com/.default\"\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&include=metadata&maxresults=2&prefix=mycontainer%2Fmyfileopenfailure.bin&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/560.3574ms, OpTime=560.442428ms) -- RESPONSE RECEIVED\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&include=metadata&maxresults=2&prefix=mycontainer%2Fmyfileopenfailure.bin&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Content-Type: application/xml\n Date: Mon, 20 Jan 2025 09:45:58 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n Vary: REDACTED\n X-Ms-Request-Id: 292e9a93-901e-0097-0c20-6bbca3000000\n X-Ms-Version: 2024-05-04\n\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (488)]: BlockBlob::getAttrUsingList : Item 0 Blob myfileopenfailure.bin\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (498)]: BlockBlob::getAttrUsingList : So far retrieved 1 objects in 1 iterations\nMon Jan 20 09:45:58 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_WARNING [block_blob.go (505)]: BlockBlob::getAttrUsingList : blob myfileopenfailure.bin does not exist\nMon Jan 20 09:46:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [block_cache.go (1643)]: BlockCache::checkDiskUsage : current disk usage : 0.003906MB 0%\nMon Jan 20 09:46:16 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_INFO [block_cache.go (1644)]: BlockCache::checkDiskUsage : current cache usage : 0%\nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : myfileopensuccess.bin\nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (515)]: BlockBlob::GetAttr : name myfileopensuccess.bin\nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (466)]: BlockBlob::getAttrUsingList : name myfileopensuccess.bin\nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (529)]: BlockBlob::List : prefix myfileopensuccess.bin, marker \nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=REDACTED&delimiter=REDACTED&include=REDACTED&maxresults=REDACTED&prefix=REDACTED&restype=REDACTED\nMon Jan 20 09:46:36 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&include=metadata&maxresults=2&prefix=mycontainer%2Fmyfileopensuccess.bin&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/1.08657163s, OpTime=1.086611169s) -- RESPONSE RECEIVED\n GET https://mystorageaccount.blob.core.windows.net/mycontainer?comp=list&delimiter=%2F&include=metadata&maxresults=2&prefix=mycontainer%2Fmyfileopensuccess.bin&restype=container\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-version: 2024-05-04\n --------------------------------------------------------------------------------\n RESPONSE Status: 200 OK\n Content-Type: application/xml\n Date: Mon, 20 Jan 2025 09:46:36 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n Vary: REDACTED\n X-Ms-Request-Id: 42f0a15c-b01e-0039-0e20-6b11b2000000\n X-Ms-Version: 2024-05-04\n\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 200\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_blob.go (488)]: BlockBlob::getAttrUsingList : Item 0 Blob myfileopensuccess.bin\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (687)]: Libfuse::libfuse_open : myfileopensuccess.bin\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (385)]: BlockCache::OpenFile : name=myfileopensuccess.bin, flags=32768, mode=-rwxrwxrwx\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (472)]: AttrCache::GetAttr : myfileopensuccess.bin\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [attr_cache.go (486)]: AttrCache::GetAttr : myfileopensuccess.bin served from cache\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [block_cache.go (397)]: BlockCache::OpenFile : Size of file handle.Size 1558\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (872)]: BlockCache::refreshBlock : Request to download 0=>myfileopensuccess.bin (index 0, prefetch false)\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (737)]: Libfuse::libfuse_open : myfileopensuccess.bin, handle 1\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : =====> Try=1 for GET https://mystorageaccount.blob.core.windows.net/mycontainer/mycontainer%2Fmyfileopensuccess.bin\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Request) : ==> OUTGOING REQUEST (Try=1)\n GET https://mystorageaccount.blob.core.windows.net/mycontainer/mycontainer%2Fmyfileopensuccess.bin\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-range: bytes=0-1557\n x-ms-version: 2024-05-04\n\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Response) : ==> REQUEST/RESPONSE (Try=1/154.041716ms, OpTime=154.077747ms) -- RESPONSE RECEIVED\n GET https://mystorageaccount.blob.core.windows.net/mycontainer/mycontainer%2Fmyfileopensuccess.bin\n Accept: application/xml\n Authorization: REDACTED\n User-Agent: Azure-Storage-Fuse/2.4.0 (Ubuntu 20.04.5 LTS) azsdk-go-azblob/v1.4.0 (go1.23.1; linux)\n x-ms-range: bytes=0-1557\n x-ms-version: 2024-05-04\n --------------------------------------------------------------------------------\n RESPONSE Status: 206 Partial Content\n Accept-Ranges: bytes\n Content-Length: 1558\n Content-Range: REDACTED\n Content-Type: application/octet-stream\n Date: Mon, 20 Jan 2025 09:46:36 GMT\n Etag: \"0x8DD36E303A8AE97\"\n Last-Modified: Fri, 17 Jan 2025 10:38:00 GMT\n Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0\n Vary: REDACTED\n X-Ms-Blob-Content-Md5: REDACTED\n X-Ms-Blob-Type: BlockBlob\n X-Ms-Creation-Time: Fri, 17 Jan 2025 10:38:00 GMT\n X-Ms-Lease-State: available\n X-Ms-Lease-Status: REDACTED\n X-Ms-Request-Id: 42f0a1a5-b01e-0039-4f20-6b11b2000000\n X-Ms-Server-Encrypted: REDACTED\n X-Ms-Version: 2024-05-04\n\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : response 206\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [utils.go (156)]: SDK(Retry) : exit due to non-retriable status code\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_DEBUG [block_cache.go (708)]: BlockCache::getBlock : Downloaded block 0 for 1=>myfileopensuccess.bin (read offset 0)\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (814)]: Libfuse::libfuse_flush : myfileopensuccess.bin, handle: 1\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (870)]: Libfuse::libfuse_release : myfileopensuccess.bin, handle: 1\nMon Jan 20 09:46:37 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (515)]: BlockCache::CloseFile : name=myfileopensuccess.bin, handle=1\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (345)]: Libfuse::libfuse_destroy : destroy\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse.go (176)]: Libfuse::Stop : Stopping component libfuse\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [libfuse_handler.go (253)]: Libfuse::destroyFuse : Destroying FUSE\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [block_cache.go (155)]: BlockCache::Stop : Stopping component block_cache\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [attr_cache.go (116)]: AttrCache::Stop : Stopping component attr_cache\nMon Jan 20 09:46:44 UTC 2025 : blobfuse2[1462] : [/tmp/blobfuse/mount] LOG_TRACE [azstorage.go (184)]: AzStorage::Stop : Stopping component azstorage\n```"}, {"author": "jainakanksha-msft", "comment": "According to the logs, the operation on myfileopensuccess.bin was successful.\nCould you please provide the exact timestamp and logs where the failure occurred so that we can investigate further? \nFor myfileopenfail.bin, Could you please list the files in container to confirm whether it exists in that directory?\nI have attempted the operation on my end, and each time it was successful."}, {"author": "listcrawler", "comment": "* I updated [my comment on directory structure](https://github.com/Azure/azure-storage-fuse/issues/1612#issuecomment-2601693090). The text was correct, but I had missed a directory level in the picture of the file tree, sorry.\n* There is a `GetAttr` failure at 09:45:58, line 416, the last hit for `myfileopenfail` in the log.\n* Yes, the files all exist. If I change to `file_chache` or mount without `--subdirectory` option, all files can be opened and read.\n* Have you verified this in WSL?"}, {"author": "jainakanksha-msft", "comment": "@listcrawler Could you please try with the latest release? Your logs do not match with the directory structure, and there is no processing at azstorage level. Looks like things are served from cache.\nAdditionally, share the directory structure as you are running the command.\nAlso run \"ls\" command to list all the files in the subdirectory once.\nThings are properly working at our end. That's why we need this input from your end."}, {"author": "listcrawler", "comment": "Have you verified this is WSL2?"}, {"author": "jainakanksha-msft", "comment": "@listcrawler yes, verified everything with WSL2."}, {"author": "listcrawler", "comment": "Right, then returning to your questions:\n\n* We're running the latest blobfuse2 version 2.4.0\n* I don't really want to share specific names publicly here, so:\n * Directory structure picture is an example, with the same layout as we have in reality.\n * Directory and blob names are somewhat longer and contain underscores.\n * The container, directory and blob names in the log are replaced with something representing their meaning. For example, `myfileopenfail.bin` in the log means, when the end-user tried to open the file, the operation failed.\n\nNot really my jurisdiction, but [the first step in the error path ](https://github.com/Azure/azure-storage-fuse/blob/32ba9ccff233795b83121f0bbab482b2a651ad14/component/attr_cache/attr_cache.go#L484) mentioned in my first post, holds the immediate question:\n\nWhy does attr_cache.go:482 say \"served from cache\" and then return a ENOENT which leads to general failure? Guessing wildly here, but wouldn't the right thing to do, to try the next component?"}, {"author": "jainakanksha-msft", "comment": "This scenario occurs only if your directory name and filename are same.\nOtherwise, I am unable to reproduce this scenario.\nIt is not merely serving from cache. \nIt's fetching the data from your storage account [for which we can see the call to backend] and processing it, which results \"no file found\" for your case."}, {"author": "listcrawler", "comment": "Briefly looking through the description of the linked bug, there may be more to this problem.\n\nWe don't really have the same name of a directory and a file. Maybe, if you removed file suffixes, there could be name clash, but we see this for all kinds of filenames.\n\nAgain, by turning blobfuse features on and off, it rather seems to be related to block/file cache, subdirectory option and flat/hierarchic storage."}, {"author": "jainakanksha-msft", "comment": "@listcrawler,\nPlease provide an example of the exact directory and file structure you are using. \nWe have tried multiple combinations but did not encounter the issue. \nTherefore, we request you to share a sample also test the same structure with same names as below from your end to see if the issue occurs.\nAnd please share your result.\n\n![Image](https://github.com/user-attachments/assets/06ee2fa7-2e04-4227-ac01-e50b606e3a32)"}, {"author": "vibhansa-msft", "comment": "PR fixing the parsing in case of subdirectory mount is merged to our main code. Next release will have the fix. Feel free to reopen if the issue persists after the next release."}]} +{"id": "issue-1611", "type": "issue", "title": "Force-refresh the cache?", "body": "Hi,\n\nWe are evaluating using blobfuse2 to cache a set of user scripts and files from their storage account and so far it has been working really well. The container will be mounted read-only across many VMs and the expectation is that the files will almost never change, so we want maximum caching. However, when the user is actively modifying the scripts and trying the changes out on a VM, we want that VM to see the newest changes. We have a mechanism to send a message to the VM that its files are out of date, so that part is taken care of. What we need is a way for the VM to make sure blobfuse2 gets those changes without waiting for the cache to expire.\n\nWe tried unmounting and remounting, which definitely picks up the latest changes. However, it also clears all cached files, which is not what we want. What we want is just to make blobfuse2 consider its cache stale, exactly as if the timeout expired, so it checks against the storage account but only downloads if some file is newer. I read in another issue about killing the process and restarting it w/ `allow-non-empty-temp`, which we would definitely do if that was the only answer, but when I tried `blobfuse2 mount` gives a `Error: directory is already mounted` message.\n\nIs there any way to tell blobfuse2 \"keep your cache but treat anything with a cache timestamp before NOW as expired, so the next time any file is accessed, it gets re-validated\"? \n\nThanks!", "author": "dougclayton", "created_at": "2025-01-17T12:04:17+00:00", "comments": [{"author": "vibhansa-msft", "comment": "After kill you need to unmount, otherwise subsequent mount will fail assuming the directory is already mounted.\nBlobfuse does not have any active way to sync the changes with container so there are only hacky solutions to what you are looking for. As you VM already gets a notification on what file has changed, what you can do is to delete the file manually from the temp-cache path. Next time when you open, blobfuse will see that the file does not exists in local cache and it will end up downloading it. However, if kernel has cached the contents then it may still keep serving the older contents, in which case unmount and then mount is the safest option,"}, {"author": "dougclayton", "comment": "Thanks for the response! That seemed pretty close to what I wanted. Unfortunately, as far as I can tell in my testing, the unmount removes the file cache for me. (It's definitely happening on unmount, not mount--I set `cleanup-on-start` to false, and I checked the cache after I unmounted but before I mounted again.) Is there any way to have blobfuse2 preserve the existing cache so that when a file is accessed, it checks the cache? Otherwise, there is no point in doing this:\n\n> what you can do is to delete the file manually from the temp-cache path. Next time when you open, blobfuse will see that the file does not exists in local cache and it will end up downloading it.\n\nBecause after unmounting blobfuse2, there is never anything in the cache, and even files that have not changed in the storage account will get re-downloaded."}, {"author": "vibhansa-msft", "comment": "Yes, as of now that's the design. On unmount we wipe out everything from the cache. On mount we do not have any in-memory information about the files that exists already in the cache and hence we download again assuming files would have changed in the container. We can take this up as a feature-request and add in our backlog. As of now there is no workaround this."}, {"author": "dougclayton", "comment": "OK, thanks for the info! We'll see what we can do with the design as it is."}]} +{"id": "issue-1610", "type": "issue", "title": "the file cache is emptied on unmount", "body": "I mount a HNS blob storage using blobfuse2, using a local cache directory.\nWhen I access a file in the mount, I can see that it is indeed copied in the cache directory.\nBut as soon as I unmount the fuse mount, the local cache directory gets deleted.\nSame behavior if I use `blobfuse2 unmount all` or `fusermount -u xxx`.\n\nIn the logs, just after the mount command, I can see this:\n```\nLOG_TRACE [libfuse_handler.go (345)]: Libfuse::libfuse_destroy : destroy\nLOG_TRACE [libfuse.go (176)]: Libfuse::Stop : Stopping component libfuse\nLOG_TRACE [libfuse_handler.go (253)]: Libfuse::destroyFuse : Destroying FUSE\nLOG_TRACE [file_cache.go (181)]: Stopping component : file_cache\nLOG_TRACE [lru_policy.go (153)]: lruPolicy::ShutdownPolicy\nLOG_TRACE [attr_cache.go (116)]: AttrCache::Stop : Stopping component attr_cache\nLOG_TRACE [azstorage.go (184)]: AzStorage::Stop : Stopping component azstorage\n```\n\nMy questions: \n - is it expected ? \n - how can I prevent that?\n\nMy use case is to have persistent file caches, so that if I restart blobfuse, it could pickup where it stopped, and avoid downloading again files in the persistent cache.\n\n\n## config\n\n```\nnonempty: true\n# nonempty: true|false \n\nlogging:\n type: base\n level: log_debug\n # type: syslog|silent|base \n # level: log_off|log_crit|log_err|log_warning|log_info|log_trace|log_debug \n # file-path: \n # max-file-size-mb: \n # file-count: \n\ncomponents:\n - libfuse\n - file_cache\n - attr_cache\n - azstorage\n \n# Libfuse configuration\nlibfuse:\n # time kernel can cache inode attributes (in sec). Default - 120 sec\n attribute-expiration-sec: 3600\n # time the kernel can cache directory listing. Default - 120 sec\n entry-expiration-sec: 3600\n # time kernel can cache attributes of non existent paths (in sec). Default - 120 sec\n negative-entry-expiration-sec: 3600\n # default-permission: 0777|0666|0644|0444 \n # fuse-trace: true|false \n # extension: \n # direct-io: true|false \n\n # \n \n## Disk cache related configuration\nfile_cache:\n # ======= required ===============\n # path to local disk cache\n path: .cache\n\n # ======= optional ===============\n # default cache eviction timeout (in sec). Default - 120 sec\n timeout-sec: 3600\n # maximum cache size allowed. Default - 80% of free disk space\n # max-size-mb: 80\n # true|false \n allow-non-empty-temp: true\n # true|false \n cleanup-on-start: false\n # sync-to-flush: true|false \n # number of seconds after which compare LMT of file in local cache and container and refresh file if container has the latest copy>. Default - 120 sec\n refresh-sec: 3600\n # ignore-sync: true|false \n # hard-limit: true|false \n \nattr_cache:\n #