added validation_config.json

scripts/world_bank/wdi/README.md

@@ -146,5 +146,22 @@ If you want to perform "only download", run the below command:
 python3 worldbank.py --mode=download
 ```
 
+### Added golden files and increased the threshold with golden checks in validation_config.json.
+
+The `GOLDENS_CHECK` validator confirms that the import includes a specific set of expected records. This is useful for verifying that critical StatVars, Places, or specific metadata combinations are consistently present in the output.
+
+The validator compares the input data (usually from the stats data source) against one or more "golden" files (MCF or CSV).
+
+If any combination of values in a golden file row is missing from the input, the validation fails. The missing golden rows are then listed in the validation report JSON.
+
+If you want to get goldens, run the below command:
+```bash
+#goldens from output csv 
+python3 validator_goldens.py   --validate_goldens_input=scripts/world_bank/wdi/output/WorldBank.csv   --generate_goldens=golden_data/golden_WorldBank.csv   --goldens_must_include="StatisticalVariable:gs://unresolved_mcfimport_validation/nl_statvars.csv"   --generate_goldens_property_sets="StatisticalVariable|observationPeriod|unit"
+
+#goldens from summary reports
+python3 validator_goldens.py --validate_goldens_input="summary_report.csv" --generate_goldens=golden_data/golden_summary_report.csv  --generate_goldens_property_sets="StatVar|NumPlaces|MinDate"
+```
+
 We highly recommend the use of the import validation tool for this import which
 you can find in https://github.com/datacommonsorg/tools/tree/master/import-validation-helper.

scripts/world_bank/wdi/golden_data/golden_WorldBank.csv

@@ -0,0 +1,58 @@
+"unit","StatisticalVariable","observationPeriod"
+"USDollar","Amount_Remittance_OutwardRemittance","P1Y"
+"USDollar","Amount_Remittance_InwardRemittance","P1Y"
+"Percent","Amount_Remittance_InwardRemittance_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y"
+"USDollar","Amount_Stock","P1Y"
+"Percent","Amount_Stock_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y"
+"","Amount_Production_ElectricityFromOilGasOrCoalSources_AsFractionOf_Amount_Production_Energy","P1Y"
+"","Amount_Production_ElectricityFromNuclearSources_AsFractionOf_Amount_Production_Energy","P1Y"
+"","Amount_Consumption_RenewableEnergy_AsFractionOf_Amount_Consumption_Energy","P1Y"
+"","Count_Product_MobileCellularSubscription_AsFractionOf_Count_Person","P1Y"
+"","Count_Person_25OrMoreYears_Female_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y"
+"","Count_Person_25OrMoreYears_Male_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y"
+"","Count_Person_25OrMoreYears_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","P1Y"
+"","Count_Person_25OrMoreYears_Female_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y"
+"","Count_Person_25OrMoreYears_Male_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y"
+"","Count_Person_25OrMoreYears_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears","P1Y"
+"","Count_Person_25OrMoreYears_Female_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y"
+"","Count_Person_25OrMoreYears_Male_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y"
+"","Count_Person_25OrMoreYears_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","P1Y"
+"","Count_Person_25OrMoreYears_Female_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y"
+"","Count_Person_25OrMoreYears_Male_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y"
+"","Count_Person_25OrMoreYears_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears","P1Y"
+"","Amount_EconomicActivity_ExpenditureActivity_TertiaryEducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government","P1Y"
+"Percent","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_Government","P1Y"
+"Percent","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y"
+"Liter","Amount_Consumption_Alcohol_15OrMoreYears_AsFractionOf_Count_Person_15OrMoreYears","P1Y"
+"Per1000LiveBirths","MortalityRate_Person_Upto4Years_AsFractionOf_Count_BirthEvent_LiveBirth","P1Y"
+"","Count_Person_15OrMoreYears_Smoking_AsFractionOf_Count_Person_15OrMoreYears","P1Y"
+"","Count_Person_15OrMoreYears_Female_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Female","P1Y"
+"","Count_Person_15OrMoreYears_Male_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Male","P1Y"
+"","Count_Person_Upto4Years_Female_Overweight_AsFractionOf_Count_Person_Upto4Years_Female","P1Y"
+"","Count_Person_Upto4Years_Male_Overweight_AsFractionOf_Count_Person_Upto4Years_Male","P1Y"
+"","Count_Person_Upto4Years_Overweight_AsFractionOf_Count_Person_Upto4Years","P1Y"
+"Per100000Females","Count_Death_IntentionalSelfHarm_Female_AsFractionOf_Count_Person_Female","P1Y"
+"Per100000Males","Count_Death_IntentionalSelfHarm_Male_AsFractionOf_Count_Person_Male","P1Y"
+"Per100000Persons","Count_Death_IntentionalSelfHarm_AsFractionOf_Count_Person","P1Y"
+"Percent","Count_Person_Upto4Years_Female_Wasting_AsFractionOf_Count_Person_Upto4Years_Female","P1Y"
+"Percent","Count_Person_Upto4Years_Male_Wasting_AsFractionOf_Count_Person_Upto4Years_Male","P1Y"
+"Percent","Count_Person_Upto4Years_Wasting_AsFractionOf_Count_Person_Upto4Years","P1Y"
+"Percent","Count_Person_Upto4Years_Female_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Female","P1Y"
+"Percent","Count_Person_Upto4Years_Male_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Male","P1Y"
+"Percent","Count_Person_Upto4Years_SevereWasting_AsFractionOf_Count_Person_Upto4Years","P1Y"
+"USDollar","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","P1Y"
+"InternationalDollar","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","P1Y"
+"","GiniIndex_EconomicActivity","P1Y"
+"","Count_Person_15To64Years_Female_InLaborForce_AsFractionOf_Count_Person_15To64Years_Female","P1Y"
+"","Count_Person_15To64Years_Male_InLaborForce_AsFractionOf_Count_Person_15To64Years_Male","P1Y"
+"","Count_Person_15To64Years_InLaborForce_AsFractionOf_Count_Person_15To64Years","P1Y"
+"","Count_Person_15OrMoreYears_InLaborForce_Female_AsFractionOf_Count_Person_InLaborForce","P1Y"
+"Per1000Persons","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","P1Y"
+"Per1000LiveBirths","Count_Death_0Years_AsFractionOf_Count_BirthEvent_LiveBirth","P1Y"
+"Year","LifeExpectancy_Person_Female","P1Y"
+"Year","LifeExpectancy_Person_Male","P1Y"
+"","Count_Person_Rural","P1Y"
+"","Count_Person_Urban","P1Y"
+"Per100000Females","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Female_AsFractionOf_Count_Person_Female","P1Y"
+"Per100000Males","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Male_AsFractionOf_Count_Person_Male","P1Y"
+"Per100000Persons","Count_CriminalActivities_MurderAndNonNegligentManslaughter_AsFractionOf_Count_Person","P1Y"

scripts/world_bank/wdi/golden_data/golden_summary_report.csv

@@ -0,0 +1,80 @@
+"MinDate","StatVar","Units","MeasurementMethods"
+"2000","Count_Death_IntentionalSelfHarm_Male_AsFractionOf_Count_Person_Male","[Per100000Males]","[]"
+"1990","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity","[InternationalDollar]","[]"
+"1983","Count_Person_Upto4Years_Wasting_AsFractionOf_Count_Person_Upto4Years","[Percent]","[JointChildMalnutritionEstimate]"
+"1994","Count_Person_25OrMoreYears_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears","[]","[]"
+"1970","Amount_Emissions_CarbonDioxide_PerCapita","[MetricTon]","[]"
+"1970","Count_Person_25OrMoreYears_Male_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]"
+"1960","LifeExpectancy_Person_Female","[Year]","[]"
+"1994","Count_Person_25OrMoreYears_Male_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]"
+"1960","Count_Death_0Years_Female_AsFractionOf_Count_BirthEvent_LiveBirth_Female","[Per1000FemaleLiveBirths]","[UnitedNationsIGMEEstimate]"
+"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_AsFractionOf_Count_Person","[Per100000Persons]","[]"
+"2000","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","[InternationalDollar, USDollar]","[]"
+"1980","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_Government","[Percent]","[]"
+"1970","Count_Person_25OrMoreYears_Male_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]"
+"1960","FertilityRate_Person_Female","[]","[]"
+"1960","Count_Person_Rural","[]","[WorldBankEstimate]"
+"1970","Count_Person_25OrMoreYears_Female_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]"
+"1960","Count_Person_Urban","[]","[WorldBankEstimate]"
+"1983","Count_Person_Upto4Years_Overweight_AsFractionOf_Count_Person_Upto4Years","[]","[]"
+"1960","LifeExpectancy_Person_Male","[Year]","[]"
+"1960","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","[Per1000Persons]","[]"
+"1960","MortalityRate_Person_Upto4Years_AsFractionOf_Count_BirthEvent_LiveBirth","[Per1000LiveBirths]","[]"
+"1960","Count_Person","[]","[]"
+"1994","Count_Person_7To14Years_Male_Employed_AsFractionOf_Count_Person_7To14Years_Male","[Percent]","[]"
+"1986","Count_Person_Upto4Years_Male_Wasting_AsFractionOf_Count_Person_Upto4Years_Male","[Percent]","[JointChildMalnutritionEstimate]"
+"1970","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[]"
+"1970","Count_Person_25OrMoreYears_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]"
+"2000","Count_Person_15OrMoreYears_Female_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Female","[]","[AgeAdjustedPrevalence]"
+"2000","Count_Person_15OrMoreYears_Smoking_AsFractionOf_Count_Person_15OrMoreYears","[]","[AgeAdjustedPrevalence]"
+"1990","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity_PerCapita","[InternationalDollar]","[]"
+"1986","Count_Person_Upto4Years_Male_Overweight_AsFractionOf_Count_Person_Upto4Years_Male","[]","[]"
+"1960","Count_Death_0Years","[]","[UnitedNationsIGMEEstimate]"
+"1970","Amount_EconomicActivity_ExpenditureActivity_TertiaryEducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government","[]","[]"
+"1986","Count_Person_Upto4Years_Male_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Male","[Percent]","[JointChildMalnutritionEstimate]"
+"1990","Amount_Consumption_Electricity_PerCapita","[KilowattHour]","[]"
+"1960","Count_Death_0Years_Male_AsFractionOf_Count_BirthEvent_LiveBirth_Male","[Per1000MaleLiveBirths]","[UnitedNationsIGMEEstimate]"
+"1990","Amount_Consumption_Energy_PerCapita","[KilogramOfOilEquivalent]","[]"
+"2000","Count_Death_IntentionalSelfHarm_Female_AsFractionOf_Count_Person_Female","[Per100000Females]","[]"
+"2000","Count_Person_15OrMoreYears_Male_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Male","[]","[AgeAdjustedPrevalence]"
+"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Male_AsFractionOf_Count_Person_Male","[Per100000Males]","[]"
+"1970","Amount_Remittance_InwardRemittance_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[WorldBankEstimate]"
+"1990","Count_Person_15To64Years_InLaborForce_AsFractionOf_Count_Person_15To64Years","[]","[]"
+"1994","Count_Person_7To14Years_Employed_AsFractionOf_Count_Person_7To14Years","[Percent]","[]"
+"1963","GiniIndex_EconomicActivity","[]","[WorldBankEstimate]"
+"1990","Count_Person_25OrMoreYears_Female_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]"
+"1990","Count_Person_25OrMoreYears_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]"
+"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Female_AsFractionOf_Count_Person_Female","[Per100000Females]","[]"
+"1990","Count_Person_15To64Years_Female_InLaborForce_AsFractionOf_Count_Person_15To64Years_Female","[]","[]"
+"1975","Amount_Stock_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[]"
+"1994","Count_Person_25OrMoreYears_Female_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]"
+"1961","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","[]","[]"
+"1960","Count_Death_AsAFractionOfCount_Person","[Per1000Persons]","[WorldBankWeightedAverage]"
+"1960","Amount_EconomicActivity_GrossDomesticProduction_Nominal","[USDollar]","[]"
+"1990","Count_Person_15To64Years_Male_InLaborForce_AsFractionOf_Count_Person_15To64Years_Male","[]","[]"
+"1970","Amount_Remittance_InwardRemittance","[USDollar]","[WorldBankEstimate]"
+"1983","Count_Person_Upto4Years_SevereWasting_AsFractionOf_Count_Person_Upto4Years","[Percent]","[JointChildMalnutritionEstimate]"
+"1970","Count_Person_25OrMoreYears_Female_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]"
+"1994","Count_Person_7To14Years_Female_Employed_AsFractionOf_Count_Person_7To14Years_Female","[Percent]","[]"
+"1990","Count_Person_25OrMoreYears_Male_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]"
+"1960","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","[USDollar]","[]"
+"2000","Amount_Consumption_Alcohol_15OrMoreYears_AsFractionOf_Count_Person_15OrMoreYears","[Liter]","[WorldHealthOrganizationEstimates]"
+"1990","Count_Person_15OrMoreYears_InLaborForce_Female_AsFractionOf_Count_Person_InLaborForce","[]","[]"
+"1990","Count_Person_ResidingLessThan5MetersAboveSeaLevel_AsFractionOf_Count_Person","[]","[]"
+"1960","Count_Product_MobileCellularSubscription_AsFractionOf_Count_Person","[]","[]"
+"1990","Count_Person_InLaborForce","[]","[InternationalLaborOrganization]"
+"2000","Count_Death_IntentionalSelfHarm_AsFractionOf_Count_Person","[Per100000Persons]","[]"
+"1960","Count_Death_0Years_AsFractionOf_Count_BirthEvent_LiveBirth","[Per1000LiveBirths]","[UnitedNationsIGMEEstimate]"
+"1986","Count_Person_Upto4Years_Female_Wasting_AsFractionOf_Count_Person_Upto4Years_Female","[Percent]","[JointChildMalnutritionEstimate]"
+"1970","Amount_Remittance_OutwardRemittance","[USDollar]","[WorldBankEstimate]"
+"1986","Count_Person_Upto4Years_Female_Overweight_AsFractionOf_Count_Person_Upto4Years_Female","[]","[]"
+"1990","Count_Person_IsInternetUser_PerCapita","[]","[]"
+"1990","Amount_Production_ElectricityFromNuclearSources_AsFractionOf_Amount_Production_Energy","[]","[]"
+"1986","Count_Person_Upto4Years_Female_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Female","[Percent]","[JointChildMalnutritionEstimate]"
+"1970","Count_Person_25OrMoreYears_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears","[]","[]"
+"1990","Amount_Production_ElectricityFromOilGasOrCoalSources_AsFractionOf_Amount_Production_Energy","[]","[]"
+"1961","GrowthRate_Count_Person","[]","[]"
+"1990","Amount_Consumption_RenewableEnergy_AsFractionOf_Amount_Consumption_Energy","[]","[]"
+"1975","Amount_Stock","[USDollar]","[]"
+"1960","LifeExpectancy_Person","[Year]","[]"
+"2000","Count_Person_20To79Years_Diabetes_AsFractionOf_Count_Person_20To79Years","[]","[]"

scripts/world_bank/wdi/manifest.json

@@ -20,7 +20,8 @@
                 "WorldBankCountries.csv",
                 "schema_csvs/WorldBankIndicators_prod.csv"
             ],
-            "cron_schedule": "0 11 * * 2"
+            "cron_schedule": "0 11 * * 2",
+            "validation_config_file": "validation_config.json"
         }
     ]
 }

scripts/world_bank/wdi/validation_config.json

@@ -0,0 +1,28 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.08
+            }
+        },
+        {
+            "rule_id": "check_goldens_output_csv",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "golden_data/golden_WorldBank.csv",
+                "input_files": "output/WorldBank.csv"
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "golden_data/golden_summary_report.csv"
+            }
+        }
+    ]
+}