[pull] master from Neo23x0:master

.github/workflows/yara-assemble.yml

@@ -21,9 +21,25 @@ jobs:
 
       # Assemble all *.yar files (except those requiring external variables)
       - name: Assemble all Yara files
-        run: "for f in $GITHUB_WORKSPACE/yara/*.yar; do if [[ (\"${f##*/}\" != \"generic_anomalies.yar\") && (\"${f##*/}\" != \"general_cloaking.yar\") && (\"${f##*/}\" != \"gen_webshells_ext_vars.yar\") && (\"${f##*/}\" != \"thor_inverse_matches.yar\") && (\"${f##*/}\" != \"yara_mixed_ext_vars.yar\") && (\"${f##*/}\" != \"configured_vulns_ext_vars.yar\") && (\"${f##*/}\" != \"gen_fake_amsi_dll.yar\") && (\"${f##*/}\" != \"expl_citrix_netscaler_adc_exploitation_cve_2023_3519.yar\") && (\"${f##*/}\" != \"yara-rules_vuln_drivers_strict_renamed.yar\") ]]; then cat $f >> signature-base.yar; fi;done"
+        run: |
+          mapfile -t excluded_rules < <(grep -E -v '^\s*(#|$)' "$GITHUB_WORKSPACE/yara/external-variable-rules.txt")
 
-      # Upload the assembled Yara artifact
+          for f in "$GITHUB_WORKSPACE"/yara/*.yar; do
+            skip_rule=false
+
+            for excluded_rule in "${excluded_rules[@]}"; do
+              if [[ "${f##*/}" == "${excluded_rule}" ]]; then
+                skip_rule=true
+                break
+              fi
+            done
+
+            if [[ "${skip_rule}" == false ]]; then
+              cat "$f" >> signature-base.yar
+            fi
+          done
+
+      # Upload the resulting Yara artifact
       - name: Upload the resulting Yara artifact
         uses: actions/upload-artifact@v4
         with:

.github/workflows/yara-syntax.yml

@@ -0,0 +1,24 @@
+name: Validate YARA Syntax
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  syntax-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check-out the repository
+        uses: actions/checkout@v4
+
+      - name: Install YARA
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y yara
+
+      - name: Show YARA version
+        run: yarac --version
+
+      - name: Validate YARA rules
+        run: bash scripts/check-yara-syntax.sh

README.md

@@ -20,18 +20,8 @@ Signature-Base is the YARA signature and IOC database for our scanners [LOKI](ht
 
 ## External Variables in YARA Rules
 
-Using the YARA rules in a tool other than [LOKI](https://github.com/Neo23x0/Loki) or [THOR Lite](https://www.nextron-systems.com/thor-lite/) will cause errors stating an `undefined identifier`. The rules that make use of external variables have been moved to the following files:
-
-- ./yara/generic_anomalies.yar
-- ./yara/general_cloaking.yar
-- ./yara/gen_webshells_ext_vars.yar
-- ./yara/thor_inverse_matches.yar
-- ./yara/yara_mixed_ext_vars.yar
-- ./yara/configured_vulns_ext_vars.yar
-- ./yara/gen_fake_amsi_dll.yar
-- ./yara/expl_citrix_netscaler_adc_exploitation_cve_2023_3519.yar
-- ./yara/yara-rules_vuln_drivers_strict_renamed.yar
-<!-- When adding entries, also update .github/workflows/yara-assemble.yml -->
+Using the YARA rules in a tool other than [LOKI](https://github.com/Neo23x0/Loki) or [THOR Lite](https://www.nextron-systems.com/thor-lite/) will cause errors stating an `undefined identifier`. The rules that make use of external variables are listed in [./yara/external-variable-rules.txt](/Users/neo/code/Workspace/signature-base/yara/external-variable-rules.txt:1). The CI syntax check and assembly workflow both use that same list.
+
 Just remove these files in case you see the above error message.
 
 ## High Quality YARA Rules Feed

scripts/check-yara-syntax.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s nullglob
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+rule_dir="${repo_root}/yara"
+external_var_rule_list="${rule_dir}/external-variable-rules.txt"
+build_dir="$(mktemp -d)"
+trap 'rm -rf "${build_dir}"' EXIT
+
+yarac_args=(
+  -w
+  -d filename=placeholder.bin
+  -d filepath=/tmp/placeholder.bin
+  -d extension=bin
+  -d "filetype=ASCII text"
+  -d filemode=0
+  -d md5=00000000000000000000000000000000
+  -d id=1
+  -d owner=root
+  -d group=root
+  -d unpack_parent=
+  -d unpack_source=
+)
+
+if [ ! -f "${external_var_rule_list}" ]; then
+  echo "External-variable rule list not found: ${external_var_rule_list}" >&2
+  exit 1
+fi
+
+mapfile -t external_var_rules < <(grep -E -v '^\s*(#|$)' "${external_var_rule_list}")
+rules=("${rule_dir}"/*.yar "${rule_dir}"/*.yara)
+
+if [ "${#rules[@]}" -eq 0 ]; then
+  echo "No YARA rules found in ${rule_dir}" >&2
+  exit 1
+fi
+
+is_external_var_rule() {
+  local rule_name
+  local allowed_rule
+
+  rule_name="$(basename "$1")"
+
+  for allowed_rule in "${external_var_rules[@]}"; do
+    if [ "${rule_name}" = "${allowed_rule}" ]; then
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+for rule_name in "${external_var_rules[@]}"; do
+  if [ ! -f "${rule_dir}/${rule_name}" ]; then
+    echo "Configured external-variable rule not found: ${rule_dir}/${rule_name}" >&2
+    exit 1
+  fi
+done
+
+strict_count=0
+external_var_count=0
+failed=0
+compiler_log="$(mktemp)"
+trap 'rm -rf "${build_dir}" "${compiler_log}"' EXIT
+
+for rule in "${rules[@]}"; do
+  if is_external_var_rule "${rule}"; then
+    external_var_count=$((external_var_count + 1))
+  else
+    strict_count=$((strict_count + 1))
+  fi
+done
+
+echo "Validating ${#rules[@]} YARA rule files (${strict_count} strict, ${external_var_count} with external variables)"
+
+for rule in "${rules[@]}"; do
+  if is_external_var_rule "${rule}"; then
+    if ! yarac "${yarac_args[@]}" "${rule}" "${build_dir}/$(basename "${rule}").compiled" >"${compiler_log}" 2>&1; then
+      echo "External-variable syntax check failed: ${rule}" >&2
+      cat "${compiler_log}" >&2
+      failed=1
+    fi
+  else
+    if ! yarac -w "${rule}" "${build_dir}/$(basename "${rule}").compiled" >"${compiler_log}" 2>&1; then
+      echo "Strict syntax check failed: ${rule}" >&2
+      cat "${compiler_log}" >&2
+      failed=1
+    fi
+  fi
+done
+
+if [ "${failed}" -ne 0 ]; then
+  exit 1
+fi
+
+echo "YARA syntax validation passed"

yara/external-variable-rules.txt

@@ -0,0 +1,13 @@
+generic_anomalies.yar
+general_cloaking.yar
+gen_webshells_ext_vars.yar
+thor_inverse_matches.yar
+yara_mixed_ext_vars.yar
+configured_vulns_ext_vars.yar
+gen_fake_amsi_dll.yar
+expl_citrix_netscaler_adc_exploitation_cve_2023_3519.yar
+expl_connectwise_screenconnect_vuln_feb24.yar
+gen_mal_3cx_compromise_mar23.yar
+gen_susp_obfuscation.yar
+gen_vcruntime140_dll_sideloading.yar
+yara-rules_vuln_drivers_strict_renamed.yar