[Wiegand] Upload parquet files on R2 using S3 SDK

Author: looorent

package.json

@@ -27,7 +27,7 @@
     "clean:d1": "rm -rf .wrangler/state/v3/d1",
     "storybook": "storybook dev -p 6006",
     "build-storybook": "storybook build",
-    "script:generate-wiegand-parquet": "tsx --max-old-space-size=4096 scripts/generate-wiegand-parquet/index.ts"
+    "script:generate-wiegand-parquet": "dotenv -e .dev.vars -- tsx --max-old-space-size=4096 scripts/generate-wiegand-parquet/index.ts"
   },
   "dependencies": {
     "@codemirror/lang-json": "6.0.2",
@@ -51,6 +51,8 @@
     "vue-router": "5.0.3"
   },
   "devDependencies": {
+    "@aws-sdk/client-s3": "3.1004.0",
+    "@aws-sdk/lib-storage": "3.1004.0",
     "@biomejs/biome": "2.4.6",
     "@cloudflare/vite-plugin": "1.26.1",
     "@storybook/addon-docs": "10.2.16",

pnpm-lock.yaml

@@ -52,9 +52,14 @@ Usage:
 Options:
   -c, --country <code>  Country code to process (${AVAILABLE_CODES})
   -u, --upload          Upload generated files to Cloudflare R2
-  -l, --local           Upload to local Miniflare R2 instead of remote
+  -l, --local           Upload to local Miniflare R2 (via wrangler) instead of remote (via S3 API)
   -h, --help            Show this help message
 
+Environment variables (required for --upload without --local):
+  TOOLBOX_CF_ACCOUNT_ID          Your Cloudflare account ID
+  TOOLBOX_CF_R2_ACCESS_KEY_ID    R2 API token access key
+  TOOLBOX_CF_R2_SECRET_ACCESS_KEY R2 API token secret key
+
 Examples:
   pnpm run script:generate-wiegand-parquet -- --country LU
   pnpm run script:generate-wiegand-parquet -- -c BE --upload

scripts/generate-wiegand-parquet/cli.ts

@@ -224,7 +224,7 @@ function* streamLuxembourgSlice(sliceIndex: number): Generator<string> {
 }
 
 // ── Netherlands ──────────────────────────────────────────────────────
-// Sidecodes 6–9:
+// Sidecodes 6-9:
 //   6: 99XXXX (45,697,600)  7: XX99XX (45,697,600)
 //   8: 9XXX99 (17,576,000)  9: X999XX (17,576,000)
 // Total: 126,547,200 — Slices: 26

scripts/generate-wiegand-parquet/countries.ts

@@ -1,7 +1,9 @@
 import { execFile } from 'node:child_process'
-import { statSync } from 'node:fs'
+import { createReadStream, statSync } from 'node:fs'
 import path from 'node:path'
 import { promisify } from 'node:util'
+import { S3Client } from '@aws-sdk/client-s3'
+import { Upload } from '@aws-sdk/lib-storage'
 import { formatFileSize } from '../../shared/utils/formatFileSize.js'
 import { logger } from '../../shared/utils/logger.js'
 
@@ -10,43 +12,106 @@ const execFileAsync = promisify(execFile)
 const R2_BUCKET = 'anpr-wiegand26-registry'
 const CONCURRENCY = 6
 
+function createS3Client(): S3Client {
+  const accountId = process.env.TOOLBOX_CF_ACCOUNT_ID
+  const accessKeyId = process.env.TOOLBOX_CF_R2_ACCESS_KEY_ID
+  const secretAccessKey = process.env.TOOLBOX_CF_R2_SECRET_ACCESS_KEY
+
+  if (!accountId || !accessKeyId || !secretAccessKey) {
+    throw new Error(
+      'Missing R2 credentials. Set TOOLBOX_CF_ACCOUNT_ID, TOOLBOX_CF_R2_ACCESS_KEY_ID, and TOOLBOX_CF_R2_SECRET_ACCESS_KEY environment variables.\n'
+        + 'Create an R2 API token at: https://dash.cloudflare.com → R2 → Manage R2 API Tokens',
+    )
+  }
+
+  return new S3Client({
+    region: 'auto',
+    endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
+    credentials: { accessKeyId, secretAccessKey },
+  })
+}
+
 export async function uploadToR2(files: string[], countryCode: string, local: boolean): Promise<void> {
-  const target = local ? 'local' : 'remote'
-  logger.info('=== Uploading %d file(s) to %s R2 bucket %s (%d concurrent) ===', files.length, target, R2_BUCKET, CONCURRENCY)
+  const target = local ? 'local (wrangler)' : 'remote (S3 API)'
+  if (local) {
+    logger.info('=== Uploading %d file(s) to %s R2 bucket %s (sequential) ===', files.length, target, R2_BUCKET)
+  } else {
+    logger.info('=== Uploading %d file(s) to %s R2 bucket %s (%d concurrent) ===', files.length, target, R2_BUCKET, CONCURRENCY)
+  }
 
   let completed = 0
 
   async function uploadWithCounter(filepath: string): Promise<void> {
-    await uploadFile(filepath, countryCode, local)
+    if (local) {
+      await uploadFileLocal(filepath, countryCode)
+    } else {
+      await uploadFileRemote(filepath, countryCode)
+    }
     completed++
     logger.info('  [%d/%d] Uploaded %s (%s)', completed, files.length, path.basename(filepath), formatFileSize(statSync(filepath).size))
   }
 
-  const queue = [...files]
-  const running = new Set<Promise<void>>()
+  if (local) {
+    for (const filepath of files) {
+      await uploadWithCounter(filepath)
+    }
+  } else {
+    const queue = [...files]
+    const running = new Set<Promise<void>>()
 
-  while (queue.length > 0 || running.size > 0) {
-    while (running.size < CONCURRENCY && queue.length > 0) {
-      const filepath = queue.shift()
-      if (filepath === undefined) {
-        break
+    while (queue.length > 0 || running.size > 0) {
+      while (running.size < CONCURRENCY && queue.length > 0) {
+        const filepath = queue.shift()
+        if (filepath === undefined) {
+          break
+        }
+        const promise = uploadWithCounter(filepath).then(() => {
+          running.delete(promise)
+        })
+        running.add(promise)
+      }
+      if (running.size > 0) {
+        await Promise.race(running)
       }
-      const promise = uploadWithCounter(filepath).then(() => {
-        running.delete(promise)
-      })
-      running.add(promise)
-    }
-    if (running.size > 0) {
-      await Promise.race(running)
     }
   }
 
   logger.info('=== Upload complete ===')
 }
 
-async function uploadFile(filepath: string, countryCode: string, local: boolean): Promise<void> {
+let s3Client: S3Client | null = null
+
+async function uploadFileRemote(filepath: string, countryCode: string): Promise<void> {
+  if (!s3Client) {
+    s3Client = createS3Client()
+  }
+
+  const key = `${countryCode}/${path.basename(filepath)}`
+
+  try {
+    const upload = new Upload({
+      client: s3Client,
+      params: {
+        Bucket: R2_BUCKET,
+        Key: key,
+        Body: createReadStream(filepath),
+        ContentType: 'application/octet-stream',
+      },
+      queueSize: 4,
+      partSize: 10 * 1024 * 1024,
+    })
+
+    await upload.done()
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error)
+    logger.error('  Failed to upload %s: %s', key, message)
+    throw error
+  }
+}
+
+async function uploadFileLocal(filepath: string, countryCode: string): Promise<void> {
   const key = `${countryCode}/${path.basename(filepath)}`
-  const args = ['r2', 'object', 'put', `${R2_BUCKET}/${key}`, '--file', filepath, ...(local ? ['--local'] : ['--remote'])]
+  const args = ['r2', 'object', 'put', `${R2_BUCKET}/${key}`, '--file', filepath, '--local']
   const wranglerBin = path.resolve(import.meta.dirname, '../../node_modules/.bin/wrangler')
 
   try {

scripts/generate-wiegand-parquet/upload.ts

@@ -29,7 +29,7 @@ export const WIEGAND_COUNTRIES: WiegandCountry[] = [
   { code: 'LU', name: 'Luxembourg', pattern: 'NNNNNN' },
   { code: 'LV', name: 'Latvia', pattern: 'LL-NNNN' },
   { code: 'MT', name: 'Malta', pattern: 'LLL-NNN' },
-  { code: 'NL', name: 'Netherlands', pattern: 'Sidecodes 6–9' },
+  { code: 'NL', name: 'Netherlands', pattern: 'Sidecodes 6-9' },
   { code: 'NO', name: 'Norway', pattern: 'LL-NNNNN' },
   { code: 'PL', name: 'Poland', pattern: 'LL-NNNNN' },
   { code: 'PT', name: 'Portugal', pattern: 'LL-NN-LL' },

shared/modules/wiegand/countries.ts

@@ -153,7 +153,7 @@ function statusClass(status: Status): string {
               :key="item"
               class="tb-row tb-items-start tb-text-xs tb-text-secondary"
             >
-              <span class="tb-text-muted tb-flex-shrink-0 tb-mt-1">–</span>
+              <span class="tb-text-muted tb-flex-shrink-0 tb-mt-1">-</span>
               <span>{{ item }}</span>
             </li>
           </ul>

src/modules/scim/ScimCompliance.vue

@@ -1,3 +1,4 @@
+import type { Wiegand26Result } from 'anpr-wiegand'
 import { decode26, decode64, encode26, encode64 } from 'anpr-wiegand'
 import type { Decode26InputFormat, WiegandDecoded26, WiegandEncoded, WiegandMode, WiegandResult } from './types'
 
@@ -33,14 +34,13 @@ async function decode26WithFormat(input: string, format: Decode26InputFormat): P
     case 'decimal': {
       const asNumber = Number(input)
       if (!/^\d+$/.test(input) || !Number.isInteger(asNumber) || asNumber < 0 || asNumber > 16_777_215) {
-        return { mode: 'error', error: 'Invalid decimal value (must be 0–16777215)' }
+        return { mode: 'error', error: 'Invalid decimal value (must be 0-16777215)' }
       }
       const hex = asNumber.toString(16).toUpperCase()
       return { mode: 'decode26', decoded: (await decode26(hex)) ?? null }
     }
-    case 'hex': {
+    case 'hex':
       return { mode: 'decode26', decoded: (await decode26(input)) ?? null }
-    }
     case 'plate': {
       const encoded = await encode26(input)
       if (!encoded) {