diff --git a/scripts/world_bank/wdi/README.md b/scripts/world_bank/wdi/README.md index ef1f0f5dc6..4dbafb89f0 100644 --- a/scripts/world_bank/wdi/README.md +++ b/scripts/world_bank/wdi/README.md @@ -146,5 +146,24 @@ If you want to perform "only download", run the below command: python3 worldbank.py --mode=download ``` +### Added golden files and increased the threshold with golden checks in validation_config.json. + +The `GOLDENS_CHECK` validator confirms that the import includes a specific set of expected records. This is useful for verifying that critical StatVars, Places, or specific metadata combinations are consistently present in the output. + +The validator compares the input data (usually from the stats data source) against one or more "golden" files (MCF or CSV). + +If any combination of values in a golden file row is missing from the input, the validation fails. The missing golden rows are then listed in the validation report JSON. + +If you want to get goldens, run the below command: +```bash +#goldens from output csv +python3 validator_goldens.py --validate_goldens_input=../../scripts/world_bank/wdi/output/WorldBank.csv --generate_goldens=golden_data/golden_observations.csv --goldens_must_include="StatisticalVariable:gs://unresolved_mcf/import_validation/nl_statvars.csv, ISO3166Alpha3:gs://unresolved_mcf/import_validation/top_100k_places.csv" --generate_goldens_property_sets="StatisticalVariable|observationPeriod|ISO3166Alpha3" +``` + +#goldens from summary reports +```bash +python3 validator_goldens.py --validate_goldens_input="summary_report.csv" --generate_goldens=golden_data/golden_summary_report.csv --generate_goldens_property_sets="StatVar|NumPlaces|MinDate" +``` + We highly recommend the use of the import validation tool for this import which you can find in https://github.com/datacommonsorg/tools/tree/master/import-validation-helper. diff --git a/scripts/world_bank/wdi/golden_data/golden_WorldBank.csv b/scripts/world_bank/wdi/golden_data/golden_WorldBank.csv new file mode 100644 index 0000000000..3b641826b6 --- /dev/null +++ b/scripts/world_bank/wdi/golden_data/golden_WorldBank.csv @@ -0,0 +1,275 @@ +"measurementMethod","StatisticalVariable","observationPeriod","ISO3166Alpha3" +"WorldBankEstimate","Amount_Remittance_OutwardRemittance","P1Y","" +"","","","dcid:Earth" +"","","","dcid:country/ABW" +"","","","dcid:country/AFG" +"","","","dcid:country/AGO" +"","","","dcid:country/ALB" +"","","","dcid:country/AND" +"","","","dcid:country/ARE" +"","","","dcid:country/ARG" +"","","","dcid:country/ARM" +"","","","dcid:country/ATG" +"","","","dcid:country/AUS" +"","","","dcid:country/AUT" +"","","","dcid:country/AZE" +"","","","dcid:country/BDI" +"","","","dcid:country/BEL" +"","","","dcid:country/BEN" +"","","","dcid:country/BFA" +"","","","dcid:country/BGD" +"","","","dcid:country/BGR" +"","","","dcid:country/BHR" +"","","","dcid:country/BHS" +"","","","dcid:country/BIH" +"","","","dcid:country/BLR" +"","","","dcid:country/BLZ" +"","","","dcid:country/BMU" +"","","","dcid:country/BOL" +"","","","dcid:country/BRA" +"","","","dcid:country/BRB" +"","","","dcid:country/BRN" +"","","","dcid:country/BTN" +"","","","dcid:country/BWA" +"","","","dcid:country/CAF" +"","","","dcid:country/CAN" +"","","","dcid:country/CHE" +"","","","dcid:country/CHL" +"","","","dcid:country/CHN" +"","","","dcid:country/CIV" +"","","","dcid:country/CMR" +"","","","dcid:country/COD" +"","","","dcid:country/COG" +"","","","dcid:country/COL" +"","","","dcid:country/COM" +"","","","dcid:country/CPV" +"","","","dcid:country/CRI" +"","","","dcid:country/CUW" +"","","","dcid:country/CYM" +"","","","dcid:country/CYP" +"","","","dcid:country/CZE" +"","","","dcid:country/DEU" +"","","","dcid:country/DJI" +"","","","dcid:country/DMA" +"","","","dcid:country/DNK" +"","","","dcid:country/DOM" +"","","","dcid:country/DZA" +"","","","dcid:country/ECU" +"","","","dcid:country/EGY" +"","","","dcid:country/ERI" +"","","","dcid:country/ESP" +"","","","dcid:country/EST" +"","","","dcid:country/ETH" +"","","","dcid:country/FIN" +"","","","dcid:country/FJI" +"","","","dcid:country/FRA" +"","","","dcid:country/FRO" +"","","","dcid:country/FSM" +"","","","dcid:country/GAB" +"","","","dcid:country/GBR" +"","","","dcid:country/GEO" +"","","","dcid:country/GHA" +"","","","dcid:country/GIN" +"","","","dcid:country/GMB" +"","","","dcid:country/GNB" +"","","","dcid:country/GNQ" +"","","","dcid:country/GRC" +"","","","dcid:country/GRD" +"","","","dcid:country/GTM" +"","","","dcid:country/GUY" +"","","","dcid:country/HKG" +"","","","dcid:country/HND" +"","","","dcid:country/HRV" +"","","","dcid:country/HTI" +"","","","dcid:country/HUN" +"","","","dcid:country/IDN" +"","","","dcid:country/IND" +"","","","dcid:country/IRL" +"","","","dcid:country/IRN" +"","","","dcid:country/IRQ" +"","","","dcid:country/ISL" +"","","","dcid:country/ISR" +"","","","dcid:country/ITA" +"","","","dcid:country/JAM" +"","","","dcid:country/JOR" +"","","","dcid:country/JPN" +"","","","dcid:country/KAZ" +"","","","dcid:country/KEN" +"","","","dcid:country/KGZ" +"","","","dcid:country/KHM" +"","","","dcid:country/KIR" +"","","","dcid:country/KNA" +"","","","dcid:country/KOR" +"","","","dcid:country/KWT" +"","","","dcid:country/LAO" +"","","","dcid:country/LBN" +"","","","dcid:country/LBR" +"","","","dcid:country/LBY" +"","","","dcid:country/LCA" +"","","","dcid:country/LKA" +"","","","dcid:country/LSO" +"","","","dcid:country/LTU" +"","","","dcid:country/LUX" +"","","","dcid:country/LVA" +"","","","dcid:country/MAC" +"","","","dcid:country/MAR" +"","","","dcid:country/MDA" +"","","","dcid:country/MDG" +"","","","dcid:country/MDV" +"","","","dcid:country/MEX" +"","","","dcid:country/MHL" +"","","","dcid:country/MKD" +"","","","dcid:country/MLI" +"","","","dcid:country/MLT" +"","","","dcid:country/MMR" +"","","","dcid:country/MNE" +"","","","dcid:country/MNG" +"","","","dcid:country/MOZ" +"","","","dcid:country/MRT" +"","","","dcid:country/MUS" +"","","","dcid:country/MWI" +"","","","dcid:country/MYS" +"","","","dcid:country/NAM" +"","","","dcid:country/NCL" +"","","","dcid:country/NER" +"","","","dcid:country/NGA" +"","","","dcid:country/NIC" +"","","","dcid:country/NLD" +"","","","dcid:country/NOR" +"","","","dcid:country/NPL" +"","","","dcid:country/NRU" +"","","","dcid:country/NZL" +"","","","dcid:country/OMN" +"","","","dcid:country/PAK" +"","","","dcid:country/PAN" +"","","","dcid:country/PER" +"","","","dcid:country/PHL" +"","","","dcid:country/PLW" +"","","","dcid:country/PNG" +"","","","dcid:country/POL" +"","","","dcid:country/PRT" +"","","","dcid:country/PRY" +"","","","dcid:country/PSE" +"","","","dcid:country/PYF" +"","","","dcid:country/QAT" +"","","","dcid:country/ROU" +"","","","dcid:country/RUS" +"","","","dcid:country/RWA" +"","","","dcid:country/SAU" +"","","","dcid:country/SDN" +"","","","dcid:country/SEN" +"","","","dcid:country/SGP" +"","","","dcid:country/SLB" +"","","","dcid:country/SLE" +"","","","dcid:country/SLV" +"","","","dcid:country/SMR" +"","","","dcid:country/SOM" +"","","","dcid:country/SRB" +"","","","dcid:country/SSD" +"","","","dcid:country/STP" +"","","","dcid:country/SUR" +"","","","dcid:country/SVK" +"","","","dcid:country/SVN" +"","","","dcid:country/SWE" +"","","","dcid:country/SWZ" +"","","","dcid:country/SXM" +"","","","dcid:country/SYC" +"","","","dcid:country/SYR" +"","","","dcid:country/TCA" +"","","","dcid:country/TCD" +"","","","dcid:country/TGO" +"","","","dcid:country/THA" +"","","","dcid:country/TJK" +"","","","dcid:country/TKM" +"","","","dcid:country/TLS" +"","","","dcid:country/TON" +"","","","dcid:country/TTO" +"","","","dcid:country/TUN" +"","","","dcid:country/TUR" +"","","","dcid:country/TUV" +"","","","dcid:country/TZA" +"","","","dcid:country/UGA" +"","","","dcid:country/UKR" +"","","","dcid:country/URY" +"","","","dcid:country/USA" +"","","","dcid:country/UZB" +"","","","dcid:country/VCT" +"","","","dcid:country/VEN" +"","","","dcid:country/VNM" +"","","","dcid:country/VUT" +"","","","dcid:country/WSM" +"","","","dcid:country/XKS" +"","","","dcid:country/YEM" +"","","","dcid:country/ZAF" +"","","","dcid:country/ZMB" +"","","","dcid:country/ZWE" +"WorldBankEstimate","Amount_Remittance_InwardRemittance","P1Y","" +"WorldBankEstimate","Amount_Remittance_InwardRemittance_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y","" +"","Amount_Stock","P1Y","" +"","","","dcid:ChannelIslands" +"","Amount_Stock_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y","" +"","Amount_Production_ElectricityFromOilGasOrCoalSources_AsFractionOf_Amount_Production_Energy","P1Y","" +"","","","dcid:country/ASM" +"","","","dcid:country/CUB" +"","","","dcid:country/GIB" +"","","","dcid:country/GRL" +"","","","dcid:country/GUM" +"","","","dcid:country/MAF" +"","","","dcid:country/PRI" +"","","","dcid:country/PRK" +"","","","dcid:country/VGB" +"","","","dcid:country/VIR" +"","Amount_Production_ElectricityFromNuclearSources_AsFractionOf_Amount_Production_Energy","P1Y","" +"","Amount_Consumption_RenewableEnergy_AsFractionOf_Amount_Consumption_Energy","P1Y","" +"","","","dcid:country/IMN" +"","","","dcid:country/LIE" +"","","","dcid:country/MNP" +"","Count_Product_MobileCellularSubscription_AsFractionOf_Count_Person","P1Y","" +"","","","dcid:country/MCO" +"","Count_Person_25OrMoreYears_Female_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y","" +"","Count_Person_25OrMoreYears_Male_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y","" +"","Count_Person_25OrMoreYears_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","P1Y","" +"","Count_Person_25OrMoreYears_Female_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y","" +"","Count_Person_25OrMoreYears_Male_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y","" +"","Count_Person_25OrMoreYears_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears","P1Y","" +"","Count_Person_25OrMoreYears_Female_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y","" +"","Count_Person_25OrMoreYears_Male_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y","" +"","Count_Person_25OrMoreYears_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","P1Y","" +"","Count_Person_25OrMoreYears_Female_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Female","P1Y","" +"","Count_Person_25OrMoreYears_Male_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Male","P1Y","" +"","Count_Person_25OrMoreYears_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears","P1Y","" +"","Amount_EconomicActivity_ExpenditureActivity_TertiaryEducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government","P1Y","" +"","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_Government","P1Y","" +"","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","P1Y","" +"WorldHealthOrganizationEstimates","Amount_Consumption_Alcohol_15OrMoreYears_AsFractionOf_Count_Person_15OrMoreYears","P1Y","" +"","MortalityRate_Person_Upto4Years_AsFractionOf_Count_BirthEvent_LiveBirth","P1Y","" +"dcs:AgeAdjustedPrevalence","Count_Person_15OrMoreYears_Smoking_AsFractionOf_Count_Person_15OrMoreYears","P1Y","" +"dcs:AgeAdjustedPrevalence","Count_Person_15OrMoreYears_Female_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Female","P1Y","" +"dcs:AgeAdjustedPrevalence","Count_Person_15OrMoreYears_Male_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Male","P1Y","" +"","Count_Person_Upto4Years_Female_Overweight_AsFractionOf_Count_Person_Upto4Years_Female","P1Y","" +"","Count_Person_Upto4Years_Male_Overweight_AsFractionOf_Count_Person_Upto4Years_Male","P1Y","" +"","Count_Person_Upto4Years_Overweight_AsFractionOf_Count_Person_Upto4Years","P1Y","" +"","Count_Death_IntentionalSelfHarm_Female_AsFractionOf_Count_Person_Female","P1Y","" +"","Count_Death_IntentionalSelfHarm_Male_AsFractionOf_Count_Person_Male","P1Y","" +"","Count_Death_IntentionalSelfHarm_AsFractionOf_Count_Person","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_Female_Wasting_AsFractionOf_Count_Person_Upto4Years_Female","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_Male_Wasting_AsFractionOf_Count_Person_Upto4Years_Male","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_Wasting_AsFractionOf_Count_Person_Upto4Years","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_Female_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Female","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_Male_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Male","P1Y","" +"JointChildMalnutritionEstimate","Count_Person_Upto4Years_SevereWasting_AsFractionOf_Count_Person_Upto4Years","P1Y","" +"","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","P1Y","" +"WorldBankEstimate","GiniIndex_EconomicActivity","P1Y","" +"","Count_Person_15To64Years_Female_InLaborForce_AsFractionOf_Count_Person_15To64Years_Female","P1Y","" +"","Count_Person_15To64Years_Male_InLaborForce_AsFractionOf_Count_Person_15To64Years_Male","P1Y","" +"","Count_Person_15To64Years_InLaborForce_AsFractionOf_Count_Person_15To64Years","P1Y","" +"","Count_Person_15OrMoreYears_InLaborForce_Female_AsFractionOf_Count_Person_InLaborForce","P1Y","" +"","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","P1Y","" +"UnitedNationsIGMEEstimate","Count_Death_0Years_AsFractionOf_Count_BirthEvent_LiveBirth","P1Y","" +"","LifeExpectancy_Person_Female","P1Y","" +"","LifeExpectancy_Person_Male","P1Y","" +"WorldBankEstimate","Count_Person_Rural","P1Y","" +"WorldBankEstimate","Count_Person_Urban","P1Y","" +"","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Female_AsFractionOf_Count_Person_Female","P1Y","" +"","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Male_AsFractionOf_Count_Person_Male","P1Y","" +"","Count_CriminalActivities_MurderAndNonNegligentManslaughter_AsFractionOf_Count_Person","P1Y","" diff --git a/scripts/world_bank/wdi/golden_data/golden_summary_report.csv b/scripts/world_bank/wdi/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..c06261a928 --- /dev/null +++ b/scripts/world_bank/wdi/golden_data/golden_summary_report.csv @@ -0,0 +1,80 @@ +"MinDate","StatVar","Units","MeasurementMethods" +"2000","Count_Death_IntentionalSelfHarm_Male_AsFractionOf_Count_Person_Male","[Per100000Males]","[]" +"1990","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity","[InternationalDollar]","[]" +"1983","Count_Person_Upto4Years_Wasting_AsFractionOf_Count_Person_Upto4Years","[Percent]","[JointChildMalnutritionEstimate]" +"1994","Count_Person_25OrMoreYears_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears","[]","[]" +"1970","Amount_Emissions_CarbonDioxide_PerCapita","[MetricTon]","[]" +"1970","Count_Person_25OrMoreYears_Male_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]" +"1960","LifeExpectancy_Person_Female","[Year]","[]" +"1994","Count_Person_25OrMoreYears_Male_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]" +"1960","Count_Death_0Years_Female_AsFractionOf_Count_BirthEvent_LiveBirth_Female","[Per1000FemaleLiveBirths]","[UnitedNationsIGMEEstimate]" +"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_AsFractionOf_Count_Person","[Per100000Persons]","[]" +"2000","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","[InternationalDollar, USDollar]","[]" +"1980","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_Government","[Percent]","[]" +"1970","Count_Person_25OrMoreYears_Male_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]" +"1960","FertilityRate_Person_Female","[]","[]" +"1960","Count_Person_Rural","[]","[WorldBankEstimate]" +"1970","Count_Person_25OrMoreYears_Female_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]" +"1960","Count_Person_Urban","[]","[WorldBankEstimate]" +"1983","Count_Person_Upto4Years_Overweight_AsFractionOf_Count_Person_Upto4Years","[]","[]" +"1960","LifeExpectancy_Person_Male","[Year]","[]" +"1960","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","[Per1000Persons]","[]" +"1960","MortalityRate_Person_Upto4Years_AsFractionOf_Count_BirthEvent_LiveBirth","[Per1000LiveBirths]","[]" +"1960","Count_Person","[]","[]" +"1994","Count_Person_7To14Years_Male_Employed_AsFractionOf_Count_Person_7To14Years_Male","[Percent]","[]" +"1986","Count_Person_Upto4Years_Male_Wasting_AsFractionOf_Count_Person_Upto4Years_Male","[Percent]","[JointChildMalnutritionEstimate]" +"1970","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[]" +"1970","Count_Person_25OrMoreYears_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]" +"2000","Count_Person_15OrMoreYears_Female_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Female","[]","[AgeAdjustedPrevalence]" +"2000","Count_Person_15OrMoreYears_Smoking_AsFractionOf_Count_Person_15OrMoreYears","[]","[AgeAdjustedPrevalence]" +"1990","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity_PerCapita","[InternationalDollar]","[]" +"1986","Count_Person_Upto4Years_Male_Overweight_AsFractionOf_Count_Person_Upto4Years_Male","[]","[]" +"1960","Count_Death_0Years","[]","[UnitedNationsIGMEEstimate]" +"1970","Amount_EconomicActivity_ExpenditureActivity_TertiaryEducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government","[]","[]" +"1986","Count_Person_Upto4Years_Male_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Male","[Percent]","[JointChildMalnutritionEstimate]" +"1990","Amount_Consumption_Electricity_PerCapita","[KilowattHour]","[]" +"1960","Count_Death_0Years_Male_AsFractionOf_Count_BirthEvent_LiveBirth_Male","[Per1000MaleLiveBirths]","[UnitedNationsIGMEEstimate]" +"1990","Amount_Consumption_Energy_PerCapita","[KilogramOfOilEquivalent]","[]" +"2000","Count_Death_IntentionalSelfHarm_Female_AsFractionOf_Count_Person_Female","[Per100000Females]","[]" +"2000","Count_Person_15OrMoreYears_Male_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Male","[]","[AgeAdjustedPrevalence]" +"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Male_AsFractionOf_Count_Person_Male","[Per100000Males]","[]" +"1970","Amount_Remittance_InwardRemittance_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[WorldBankEstimate]" +"1990","Count_Person_15To64Years_InLaborForce_AsFractionOf_Count_Person_15To64Years","[]","[]" +"1994","Count_Person_7To14Years_Employed_AsFractionOf_Count_Person_7To14Years","[Percent]","[]" +"1963","GiniIndex_EconomicActivity","[]","[WorldBankEstimate]" +"1990","Count_Person_25OrMoreYears_Female_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]" +"1990","Count_Person_25OrMoreYears_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]" +"1990","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Female_AsFractionOf_Count_Person_Female","[Per100000Females]","[]" +"1990","Count_Person_15To64Years_Female_InLaborForce_AsFractionOf_Count_Person_15To64Years_Female","[]","[]" +"1975","Amount_Stock_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[Percent]","[]" +"1994","Count_Person_25OrMoreYears_Female_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]" +"1961","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","[]","[]" +"1960","Count_Death_AsAFractionOfCount_Person","[Per1000Persons]","[WorldBankWeightedAverage]" +"1960","Amount_EconomicActivity_GrossDomesticProduction_Nominal","[USDollar]","[]" +"1990","Count_Person_15To64Years_Male_InLaborForce_AsFractionOf_Count_Person_15To64Years_Male","[]","[]" +"1970","Amount_Remittance_InwardRemittance","[USDollar]","[WorldBankEstimate]" +"1983","Count_Person_Upto4Years_SevereWasting_AsFractionOf_Count_Person_Upto4Years","[Percent]","[JointChildMalnutritionEstimate]" +"1970","Count_Person_25OrMoreYears_Female_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]" +"1994","Count_Person_7To14Years_Female_Employed_AsFractionOf_Count_Person_7To14Years_Female","[Percent]","[]" +"1990","Count_Person_25OrMoreYears_Male_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]" +"1960","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","[USDollar]","[]" +"2000","Amount_Consumption_Alcohol_15OrMoreYears_AsFractionOf_Count_Person_15OrMoreYears","[Liter]","[WorldHealthOrganizationEstimates]" +"1990","Count_Person_15OrMoreYears_InLaborForce_Female_AsFractionOf_Count_Person_InLaborForce","[]","[]" +"1990","Count_Person_ResidingLessThan5MetersAboveSeaLevel_AsFractionOf_Count_Person","[]","[]" +"1960","Count_Product_MobileCellularSubscription_AsFractionOf_Count_Person","[]","[]" +"1990","Count_Person_InLaborForce","[]","[InternationalLaborOrganization]" +"2000","Count_Death_IntentionalSelfHarm_AsFractionOf_Count_Person","[Per100000Persons]","[]" +"1960","Count_Death_0Years_AsFractionOf_Count_BirthEvent_LiveBirth","[Per1000LiveBirths]","[UnitedNationsIGMEEstimate]" +"1986","Count_Person_Upto4Years_Female_Wasting_AsFractionOf_Count_Person_Upto4Years_Female","[Percent]","[JointChildMalnutritionEstimate]" +"1970","Amount_Remittance_OutwardRemittance","[USDollar]","[WorldBankEstimate]" +"1986","Count_Person_Upto4Years_Female_Overweight_AsFractionOf_Count_Person_Upto4Years_Female","[]","[]" +"1990","Count_Person_IsInternetUser_PerCapita","[]","[]" +"1990","Amount_Production_ElectricityFromNuclearSources_AsFractionOf_Amount_Production_Energy","[]","[]" +"1986","Count_Person_Upto4Years_Female_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Female","[Percent]","[JointChildMalnutritionEstimate]" +"1970","Count_Person_25OrMoreYears_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears","[]","[]" +"1990","Amount_Production_ElectricityFromOilGasOrCoalSources_AsFractionOf_Amount_Production_Energy","[]","[]" +"1961","GrowthRate_Count_Person","[]","[]" +"1990","Amount_Consumption_RenewableEnergy_AsFractionOf_Amount_Consumption_Energy","[]","[]" +"1975","Amount_Stock","[USDollar]","[]" +"1960","LifeExpectancy_Person","[Year]","[]" +"2000","Count_Person_20To79Years_Diabetes_AsFractionOf_Count_Person_20To79Years","[]","[]" diff --git a/scripts/world_bank/wdi/manifest.json b/scripts/world_bank/wdi/manifest.json index bc3927141e..eb427c0472 100644 --- a/scripts/world_bank/wdi/manifest.json +++ b/scripts/world_bank/wdi/manifest.json @@ -20,7 +20,8 @@ "WorldBankCountries.csv", "schema_csvs/WorldBankIndicators_prod.csv" ], - "cron_schedule": "0 11 * * 2" + "cron_schedule": "0 11 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json new file mode 100644 index 0000000000..31b81387a7 --- /dev/null +++ b/scripts/world_bank/wdi/validation_config.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.08 + } + }, + { + "rule_id": "check_goldens_output_csv", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_WorldBank.csv", + "input_files": "output/WorldBank.csv" + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_summary_report.csv" + } + } + ] +} \ No newline at end of file diff --git a/tools/import_validation/Validations.md b/tools/import_validation/Validations.md index 4efebb3a55..d46ece74fc 100644 --- a/tools/import_validation/Validations.md +++ b/tools/import_validation/Validations.md @@ -72,6 +72,8 @@ To generate goldens for the summary_report.csv to verify that all the expected StatVars are generated with the corresponding number of places and dates, run the following: +This will compare the golden files using summary_report.csv as the default input: + ```shell python3 validator_goldens.py \ --validate_goldens_input=summary_report.csv \