Merge pull request #22 from thaddeus/kv-request-queue

Kv request queue

Author: Razzmatazzz

.github/workflows/branch-deploy.yml

@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: github/branch-deploy@v9.4.0
+      - uses: github/branch-deploy@v9.7.0
         id: branch-deploy
         with:
           admins: the-hideout/core-contributors

.github/workflows/deploy.yml

@@ -66,7 +66,7 @@ jobs:
     steps:
       # https://github.com/github/branch-deploy/blob/d3c24bd92505e623615b75ffdfac5ed5259adbdb/docs/merge-commit-strategy.md
       - name: deployment check
-        uses: github/branch-deploy@v9.4.0
+        uses: github/branch-deploy@v9.7.0
         id: deployment-check
         with:
           merge_deploy_mode: "true"

.github/workflows/unlock-on-merge.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: unlock on merge
-        uses: github/branch-deploy@v9.4.0
+        uses: github/branch-deploy@v9.7.0
         id: unlock-on-merge
         with:
           unlock_on_merge_mode: "true" # <-- indicates that this is the "Unlock on Merge Mode" workflow

http/cloudflare-kv.mjs

@@ -0,0 +1,61 @@
+import { EventEmitter } from 'node:events';
+
+import fetchWithTimeout from '../utils/fetch-with-timeout.mjs';
+
+const completeEmitter = new EventEmitter();
+
+const accountId = '424ad63426a1ae47d559873f929eb9fc';
+
+const productionNamespaceId = '2e6feba88a9e4097b6d2209191ed4ae5';
+const devNameSpaceID = '17fd725f04984e408d4a70b37c817171';
+
+const requestLimit = 6;
+
+let pending = [];
+const queue = [];
+
+const checkQueue = async () => {
+    if (pending.length >= requestLimit) {
+        return;
+    }
+    if (queue.length < 1) {
+        return;
+    }
+    const kvName = queue.shift();
+    pending.push(kvName);
+
+    const namespaceId = process.env.ENVIRONMENT === 'production' ? productionNamespaceId : devNameSpaceID;
+    const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${kvName}`;
+    let response;
+    try {
+        response = await fetchWithTimeout(url, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${process.env.CLOUDFLARE_TOKEN}`,
+            },
+            timeout: 9000,
+        });
+        completeEmitter.emit(kvName, response);
+    } catch (error) {
+        //response = new Response(null, {status: 500, statusText: error.message});
+        queue.unshift(kvName);
+    } finally {
+        pending = pending.filter(kv => kv !== kvName);
+    }
+    checkQueue();
+};
+
+const cloudflareKv = {
+    get: async (kvName) => {
+        return new Promise((resolve) => {
+            completeEmitter.once(kvName, resolve);
+            if (!pending.includes(kvName) && !queue.includes(kvName)) {
+                queue.push(kvName);
+            }
+            checkQueue();
+        });
+    },
+};
+
+export default cloudflareKv;

http/env-binding.mjs

@@ -6,11 +6,7 @@ import { EventEmitter } from 'node:events';
 import { v4 as uuidv4} from 'uuid';
 
 import cacheMachine from '../utils/cache-machine.mjs';
-
-const accountId = '424ad63426a1ae47d559873f929eb9fc';
-
-const productionNamespaceId = '2e6feba88a9e4097b6d2209191ed4ae5';
-const devNameSpaceID = '17fd725f04984e408d4a70b37c817171';
+import cloudflareKv from './cloudflare-kv.mjs';
 
 const emitter = new EventEmitter();
 
@@ -43,15 +39,7 @@ async function messageParentProcess(message) {
 }
 
 async function getDataPrimary(kvName, format) {
-    const namespaceId = process.env.ENVIRONMENT === 'production' ? productionNamespaceId : devNameSpaceID;
-    const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${kvName}`;
-    const response = await fetch(url, {
-        method: 'GET',
-        headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${process.env.CLOUDFLARE_TOKEN}`,
-        },
-    });
+    const response = await cloudflareKv.get(kvName);
     if (response.status === 404) {
         return null;
     }
@@ -68,7 +56,7 @@ async function getDataPrimary(kvName, format) {
 }
 
 async function getDataWorker(kvName, format) {
-    return messageParentProcess({action: 'getKv', kvName});
+    return messageParentProcess({action: 'getKv', kvName, timeout: 25000});
 }
 
 const DATA_CACHE = {

http/package-lock.json

@@ -17,12 +17,12 @@
         "nodemon": "^3.1.3"
     },
     "dependencies": {
-        "@graphql-tools/merge": "9.0.4",
+        "@graphql-tools/merge": "9.0.7",
+        "@graphql-tools/schema": "10.0.6",
         "@sentry/cli": "^2.32.2",
         "@sentry/node": "^8.18.0",
         "@sentry/profiling-node": "^8.18.0",
-        "@graphql-tools/schema": "10.0.4",
-        "graphql-yoga": "^5.6.3",
+        "graphql-yoga": "^5.7.0",
         "dotenv": "^16.4.5",
         "uuid": "^10.0.0"
     },

http/package.json

@@ -0,0 +1,40 @@
+import 'dotenv/config';
+
+import DataAPI from '../datasources/index.mjs';
+import cloudflareKv from './cloudflare-kv.mjs';
+
+const data = new DataAPI();
+
+const getKv = async (kvName) => {
+    const response = await cloudflareKv.get(kvName);
+    if (response.status !== 200) {
+        console.error('error', kvName, `${response.status} ${response.statusText}`);
+        return;
+    }
+    console.log(response.status, kvName, (await response.text()).length);
+};
+
+for (const workerName in data.worker) {
+    const worker = data.worker[workerName];
+    for (const gameMode of worker.gameModes) {
+        let kvName = worker.kvName;
+        let suffix = '';
+        if (gameMode !== 'regular') {
+            suffix = `_${gameMode}`;
+        }
+        try {
+            if (worker.kvs) {
+                for (const hexKey in worker.kvs) {
+                    const splitWorker = worker.kvs[hexKey];
+                    const fullKvName = `${splitWorker.kvName}${suffix}`;
+                    getKv(fullKvName);
+                }
+            } else {
+                const fullKvName = `${kvName}${suffix}`;
+                getKv(fullKvName);
+            }
+        } catch (error) {
+            console.error(kvName, gameMode, error);
+        }
+    }
+}