fix:add s3 proxy when env config (#152)

* fix:add s3 proxy when env config

* fix: add process.env and setup fetch proxy

* chore: adjust the code

* fix: use undiciFetch instead of fetch

---------

Co-authored-by: FinleyGe <[email protected]>

Author: dingzhenznen

bun.lock

@@ -23,9 +23,12 @@
         "comlink": "^4.4.2",
         "date-fns": "^4.1.0",
         "express": "^5.1.0",
+        "http-proxy-agent": "^7.0.2",
+        "https-proxy-agent": "^7.0.6",
         "json5": "^2.2.3",
         "minio": "^8.0.5",
         "nanoid": "^5.1.5",
+        "undici": "^7.13.0",
         "uuid": "^11.1.0",
         "zod": "^3.24.3",
       },
@@ -429,7 +432,7 @@
     },
     "sdk": {
       "name": "@fastgpt-sdk/plugin",
-      "version": "0.1.3",
+      "version": "0.1.7",
       "dependencies": {
         "@fortaine/fetch-event-source": "^3.0.6",
         "@ts-rest/core": "^3.52.1",
@@ -1891,7 +1894,7 @@
 
     "ufo": ["[email protected]", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],
 
-    "undici": ["undici@7.12.0", "", {}, "sha512-GrKEsc3ughskmGA9jevVlIOPMiiAHJ4OFUtaAH+NhfTUSiZ1wMPIQqQvAJUrJspFXJt3EBWgpAeoHEDVT1IBug=="],
+    "undici": ["undici@7.13.0", "", {}, "sha512-l+zSMssRqrzDcb3fjMkjjLGmuiiK2pMIcV++mJaAc9vhjSGpvM7h43QgP+OAMb1GImHmbPyG2tBXeuyG5iY4gA=="],
 
     "undici-types": ["[email protected]", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="],
 
@@ -2067,6 +2070,8 @@
 
     "bl/readable-stream": ["[email protected]", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="],
 
+    "cheerio/undici": ["[email protected]", "", {}, "sha512-GrKEsc3ughskmGA9jevVlIOPMiiAHJ4OFUtaAH+NhfTUSiZ1wMPIQqQvAJUrJspFXJt3EBWgpAeoHEDVT1IBug=="],
+
     "cli-truncate/string-width": ["[email protected]", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
 
     "cliui/strip-ansi": ["[email protected]", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],

package.json

@@ -36,9 +36,12 @@
     "comlink": "^4.4.2",
     "date-fns": "^4.1.0",
     "express": "^5.1.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "json5": "^2.2.3",
     "minio": "^8.0.5",
     "nanoid": "^5.1.5",
+    "undici": "^7.13.0",
     "uuid": "^11.1.0",
     "zod": "^3.24.3"
   },

src/index.ts

@@ -7,6 +7,7 @@ import { isProd } from './constants';
 import { initS3Server } from './s3/config';
 import { connectSignoz } from './utils/signoz';
 import { initModels } from '@model/init';
+import { setupProxy } from './utils/setupProxy';
 
 const app = express().use(
   express.json(),
@@ -19,6 +20,7 @@ connectSignoz();
 // System
 initOpenAPI(app);
 initRouter(app);
+setupProxy();
 
 // DB
 try {

src/s3/controller.ts

@@ -7,6 +7,8 @@ import { z } from 'zod';
 import { addLog } from '@/utils/log';
 import { getErrText } from '@tool/utils/err';
 import { catchError } from '@/utils/catch';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
 
 export const FileInputSchema = z
   .object({
@@ -52,7 +54,12 @@ export class S3Service {
       port: this.config.port,
       useSSL: this.config.useSSL,
       accessKey: this.config.accessKey,
-      secretKey: this.config.secretKey
+      secretKey: this.config.secretKey,
+      transportAgent: process.env.HTTP_PROXY
+        ? new HttpProxyAgent(process.env.HTTP_PROXY)
+        : process.env.HTTPS_PROXY
+          ? new HttpsProxyAgent(process.env.HTTPS_PROXY)
+          : undefined
     });
   }
 

src/utils/setupProxy.ts

@@ -0,0 +1,23 @@
+import { ProxyAgent, setGlobalDispatcher, fetch as undiciFetch } from 'undici';
+import { addLog } from './log';
+import { isProd } from '@/constants';
+const httpProxy = process.env.HTTP_PROXY;
+const httpsProxy = process.env.HTTPS_PROXY;
+
+export function setupProxy() {
+  const proxy = httpProxy || httpsProxy;
+  if (proxy) {
+    const proxyAgent = new ProxyAgent(proxy);
+    setGlobalDispatcher(proxyAgent);
+
+    // Replace global fetch with undici's fetch to ensure proxy is used
+    if (isProd) {
+      // Node
+      global.fetch = ((input: any, init: any) => {
+        return undiciFetch(input, init);
+      }) as any;
+    }
+
+    addLog.info(`Using proxy: ${proxy}`);
+  }
+}

src/worker/index.ts

@@ -93,7 +93,10 @@ export class WorkerPool<Props = Record<string, unknown>, Response = unknown> {
     // Create a new worker and push it queue.
     const workerId = `${Date.now()}${Math.random()}`;
     const worker = new Worker('./worker.js', {
-      env: {},
+      env: {
+        ...(process.env.HTTP_PROXY ? { HTTP_PROXY: process.env.HTTP_PROXY } : {}),
+        ...(process.env.HTTPS_PROXY ? { HTTPS_PROXY: process.env.HTTPS_PROXY } : {})
+      },
       resourceLimits: {
         maxOldGenerationSizeMb: parseInt(process.env.MAX_MEMORYMB || '1024')
       }

src/worker/worker.ts

@@ -4,6 +4,8 @@ import { LoadToolsByFilename } from '@tool/init';
 import { isProd } from '@/constants';
 import { getErrText } from '@tool/utils/err';
 import type { Main2WorkerMessageType } from './type';
+import { setupProxy } from '@/utils/setupProxy';
+setupProxy();
 
 // rewrite console.log to send to parent
 console.log = (...args: any[]) => {