[CI] Add new script + cronjob to amend pull request data

premerge/bigquery_schema/llvm_pull_requests_table_schema.json

@@ -59,5 +59,11 @@
     "type": "STRING",
     "mode": "REPEATED",
     "description": "GitHub usernames of Users requested to review the pull request."
+  },
+  {
+    "name": "is_stale_data",
+    "type": "BOOLEAN",
+    "mode": "NULLABLE",
+    "description": "Whether or not this data is stale, outdated. Stale data will not be amended by daily cronjobs."
   }
 ]

premerge/cronjobs/amend_pull_request_data_cronjob.yaml

@@ -0,0 +1,37 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: amend-pull-request-data-cronjob
+  namespace: operational-metrics
+spec:
+  # Every 2 hours
+  # Half of the lookback window in amend_pull_request_data.py
+  schedule: "0 */2 * * *"
+  timeZone: "Etc/UTC"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: operational-metrics-ksa
+          containers:
+          - name: amend-pull-request-data
+            image: ghcr.io/llvm/operations-metrics:latest
+            command: ["python3", "amend_pull_request_data.py"]
+            env:
+            - name: GITHUB_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: operational-metrics-secrets
+                  key: github-token
+            resources:
+              requests:
+                cpu: "250m"
+                # We explicitly use Mi here instead of a decimal number of
+                # Gi because k8s will automatically convert to Mi which
+                # terraform then thinks differs from what it intended to apply.
+                memory: "512Mi"
+              limits:
+                cpu: "2"
+                memory: "2Gi"
+          restartPolicy: OnFailure

premerge/operational_metrics_cronjob.yaml → premerge/cronjobs/process_llvm_commits_cronjob.yaml

@@ -1,8 +1,7 @@
-# operational_metrics_cronjob.yaml
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: operational-metrics-cronjob
+  name: process-llvm-commits-cronjob
   namespace: operational-metrics
 spec:
   # Midnight PDT
@@ -14,11 +13,10 @@ spec:
       template:
         spec:
           serviceAccountName: operational-metrics-ksa
-          nodeSelector:
-            iam.gke.io/gke-metadata-server-enabled: "true"
           containers:
           - name: process-llvm-commits
             image: ghcr.io/llvm/operations-metrics:latest
+            command: ["python3", "process_llvm_commits.py"]
             env:
             - name: GITHUB_TOKEN
               valueFrom:

premerge/main.tf

@@ -323,8 +323,8 @@ resource "kubernetes_secret" "operational_metrics_secrets" {
   depends_on = [kubernetes_namespace.operational_metrics]
 }
 
-resource "kubernetes_manifest" "operational_metrics_cronjob" {
-  manifest = yamldecode(file("operational_metrics_cronjob.yaml"))
+resource "kubernetes_manifest" "process_llvm_commits_cronjob" {
+  manifest = yamldecode(file("./cronjobs/process_llvm_commits_cronjob.yaml"))
   provider = kubernetes.llvm-premerge-us-central
 
   depends_on = [
@@ -334,6 +334,17 @@ resource "kubernetes_manifest" "operational_metrics_cronjob" {
   ]
 }
 
+resource "kubernetes_manifest" "amend_pull_request_data_cronjob" {
+  manifest = yamldecode(file("./cronjobs/amend_pull_request_data_cronjob.yaml"))
+  provider = kubernetes.llvm-premerge-us-central
+
+  depends_on = [
+    kubernetes_namespace.operational_metrics,
+    kubernetes_secret.operational_metrics_secrets,
+    kubernetes_service_account.operational_metrics_ksa
+  ]
+}
+
 # BigQuery dataset and table resources
 resource "google_bigquery_dataset" "operational_metrics_dataset" {
   dataset_id  = "operational_metrics"

premerge/ops-container/Dockerfile

@@ -2,6 +2,5 @@ FROM docker.io/python:3.12
 
 COPY requirements.lock.txt ./
 RUN pip3 install --no-cache-dir -r requirements.lock.txt
-COPY process_llvm_commits.py operational_metrics_lib.py ./
+COPY process_llvm_commits.py amend_pull_request_data.py operational_metrics_lib.py ./
 
-CMD ["python3", "process_llvm_commits.py"]