[Wiegand] --force argument in script to upload parquet files to R2

Author: looorent

scripts/generate-wiegand-parquet/cli.ts

@@ -7,6 +7,7 @@ export interface CliConfig {
   country: Country
   upload: boolean
   local: boolean
+  force: boolean
 }
 
 export function parseCli(): CliConfig {
@@ -15,6 +16,7 @@ export function parseCli(): CliConfig {
       country: { type: 'string', short: 'c' },
       upload: { type: 'boolean', short: 'u', default: false },
       local: { type: 'boolean', short: 'l', default: false },
+      force: { type: 'boolean', short: 'f', default: false },
       help: { type: 'boolean', short: 'h', default: false },
     },
     strict: true,
@@ -34,6 +36,7 @@ export function parseCli(): CliConfig {
     country: countriesByCode[countryCode],
     upload: values.upload ?? false,
     local: values.local ?? false,
+    force: values.force ?? false,
   }
 }
 
@@ -46,13 +49,14 @@ Wiegand26 -> License Plate parquet generator
 Generates parquet files mapping Wiegand26 decimal values to license plate texts.
 
 Usage:
-  pnpm run script:generate-wiegand-parquet -- --country <code> [--upload] [--local]
+  pnpm run script:generate-wiegand-parquet -- --country <code> [--upload] [--local] [--force]
   pnpm run script:generate-wiegand-parquet -- --help
 
 Options:
   -c, --country <code>  Country code to process (${AVAILABLE_CODES})
   -u, --upload          Upload generated files to Cloudflare R2
   -l, --local           Upload to local Miniflare R2 (via wrangler) instead of remote (via S3 API)
+  -f, --force           Re-upload files even if they already exist on R2
   -h, --help            Show this help message
 
 Environment variables (required for --upload without --local):

scripts/generate-wiegand-parquet/index.ts

@@ -10,13 +10,13 @@ import { uploadToR2 } from './upload.js'
 const OUTPUT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'output')
 const TEMP_DIR = path.join(OUTPUT_DIR, 'tmp')
 
-const { country, upload, local } = parseCli()
+const { country, upload, local, force } = parseCli()
 
 async function main(): Promise<void> {
   logger.info('=== Wiegand26 -> License Plate parquet generator ===')
   logger.info('Output directory:', OUTPUT_DIR)
   logger.info('Country:', country.code)
-  logger.info('Upload to R2:', upload, local ? '(local)' : '(remote)')
+  logger.info('Upload to R2:', upload, local ? '(local)' : '(remote)', force ? '(force)' : '')
   logger.info('Ranges:', computeRanges().length)
 
   mkdirSync(OUTPUT_DIR, { recursive: true })
@@ -26,7 +26,7 @@ async function main(): Promise<void> {
   rmSync(TEMP_DIR, { recursive: true })
 
   if (upload) {
-    await uploadToR2(files, country.code, local)
+    await uploadToR2(files, country.code, local, force)
   } else {
     logger.info('Skipping R2 upload (use --upload to enable)')
   }

scripts/generate-wiegand-parquet/upload.ts

@@ -1,12 +1,13 @@
-import { execFile } from 'node:child_process'
+import { exec, execFile } from 'node:child_process'
 import { createReadStream, statSync } from 'node:fs'
 import path from 'node:path'
 import { promisify } from 'node:util'
-import { S3Client } from '@aws-sdk/client-s3'
+import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
 import { formatFileSize } from '../../shared/utils/formatFileSize.js'
 import { logger } from '../../shared/utils/logger.js'
 
+const execAsync = promisify(exec)
 const execFileAsync = promisify(execFile)
 
 const R2_BUCKET = 'anpr-wiegand26-registry'
@@ -31,7 +32,7 @@ function createS3Client(): S3Client {
   })
 }
 
-export async function uploadToR2(files: string[], countryCode: string, local: boolean): Promise<void> {
+export async function uploadToR2(files: string[], countryCode: string, local: boolean, force: boolean): Promise<void> {
   const target = local ? 'local (wrangler)' : 'remote (S3 API)'
   if (local) {
     logger.info('=== Uploading %d file(s) to %s R2 bucket %s (sequential) ===', files.length, target, R2_BUCKET)
@@ -40,15 +41,36 @@ export async function uploadToR2(files: string[], countryCode: string, local: bo
   }
 
   let completed = 0
+  let skipped = 0
 
   async function uploadWithCounter(filepath: string): Promise<void> {
+    const key = `${countryCode}/${path.basename(filepath)}`
+
+    if (!force && !local) {
+      const exists = await objectExistsRemote(key)
+      if (exists) {
+        skipped++
+        logger.info('  [skip] %s already exists on R2', key)
+        return
+      }
+    }
+
+    if (!force && local) {
+      const exists = await objectExistsLocal(key)
+      if (exists) {
+        skipped++
+        logger.info('  [skip] %s already exists on local R2', key)
+        return
+      }
+    }
+
     if (local) {
       await uploadFileLocal(filepath, countryCode)
     } else {
       await uploadFileRemote(filepath, countryCode)
     }
     completed++
-    logger.info('  [%d/%d] Uploaded %s (%s)', completed, files.length, path.basename(filepath), formatFileSize(statSync(filepath).size))
+    logger.info('  [%d/%d] Uploaded %s (%s)', completed, files.length - skipped, path.basename(filepath), formatFileSize(statSync(filepath).size))
   }
 
   if (local) {
@@ -76,21 +98,51 @@ export async function uploadToR2(files: string[], countryCode: string, local: bo
     }
   }
 
-  logger.info('=== Upload complete ===')
+  if (skipped > 0) {
+    logger.info('=== Upload complete (%d uploaded, %d skipped) ===', completed, skipped)
+  } else {
+    logger.info('=== Upload complete ===')
+  }
 }
 
 let s3Client: S3Client | null = null
 
-async function uploadFileRemote(filepath: string, countryCode: string): Promise<void> {
+function getS3Client(): S3Client {
   if (!s3Client) {
     s3Client = createS3Client()
   }
+  return s3Client
+}
+
+async function objectExistsRemote(key: string): Promise<boolean> {
+  const client = getS3Client()
+  try {
+    await client.send(new HeadObjectCommand({ Bucket: R2_BUCKET, Key: key }))
+    return true
+  } catch {
+    return false
+  }
+}
 
+async function objectExistsLocal(key: string): Promise<boolean> {
+  const wranglerBin = path.resolve(import.meta.dirname, '../../node_modules/.bin/wrangler')
+  try {
+    await execAsync(`"${wranglerBin}" r2 object get "${R2_BUCKET}/${key}" --local --pipe > /dev/null`, {
+      cwd: path.resolve(import.meta.dirname, '../..'),
+    })
+    return true
+  } catch {
+    return false
+  }
+}
+
+async function uploadFileRemote(filepath: string, countryCode: string): Promise<void> {
+  const client = getS3Client()
   const key = `${countryCode}/${path.basename(filepath)}`
 
   try {
     const upload = new Upload({
-      client: s3Client,
+      client,
       params: {
         Bucket: R2_BUCKET,
         Key: key,