Skip to content

Commit cc934e4

Browse files
AbeshithAbeshith
committed
Update CI/CD and gitignore for DVC integration
1 parent 86d13fb commit cc934e4

14 files changed

Lines changed: 6627 additions & 309 deletions

File tree

‎.github/workflows/ci-cd.yml‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,33 @@ jobs:
5050
run: |
5151
python -m pip install --upgrade pip
5252
pip install -r requirements.txt
53-
pip install flake8 pytest black isort bandit
53+
pip install flake8 pytest black isort bandit dvc
54+
55+
- name: Pull DVC artifacts
56+
run: |
57+
# Configure DVC remote (you'll need to set this up)
58+
# dvc remote add -d myremote s3://your-bucket/path
59+
# dvc pull || echo "No DVC remote configured, skipping model pull"
60+
61+
# For now, ensure model directory exists
62+
mkdir -p artifacts/model_trainer
63+
64+
# Create a dummy model if none exists (for CI/CD)
65+
python -c "
66+
import pickle
67+
import os
68+
from sklearn.ensemble import RandomForestClassifier
69+
70+
model_path = 'artifacts/model_trainer/model.pkl'
71+
if not os.path.exists(model_path):
72+
print('Creating dummy model for CI/CD')
73+
dummy_model = RandomForestClassifier(n_estimators=10)
74+
os.makedirs(os.path.dirname(model_path), exist_ok=True)
75+
with open(model_path, 'wb') as f:
76+
pickle.dump(dummy_model, f)
77+
else:
78+
print('Model already exists')
79+
"
5480
5581
- name: Code formatting check
5682
run: |
@@ -103,6 +129,24 @@ jobs:
103129
print(f'⚠ {file} not found')
104130
"
105131
132+
- name: Validate DVC pipeline
133+
run: |
134+
# Check DVC pipeline syntax
135+
dvc dag --ascii || echo "DVC pipeline validation skipped"
136+
137+
# Validate DVC stages
138+
if [ -f "dvc.yaml" ]; then
139+
echo "✓ dvc.yaml found"
140+
python -c "
141+
import yaml
142+
with open('dvc.yaml', 'r') as f:
143+
dvc_config = yaml.safe_load(f)
144+
print(f'✓ DVC pipeline has {len(dvc_config.get(\"stages\", {}))} stages')
145+
"
146+
else
147+
echo "⚠ dvc.yaml not found"
148+
fi
149+
106150
- name: Check model artifacts
107151
run: |
108152
python -c "

‎.gitignore‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,14 @@
11
__pycache__/
22
Observability.md
33
.venv/
4-
uv.lock
54
.__pycache__
65

76

8-
# DVC managed artifacts
7+
# DVC managed artifacts (all artifacts handled by DVC)
98
artifacts/
10-
artifacts/data_ingestion/train_data/
11-
artifacts/data_ingestion/test_data/
12-
artifacts/data_transformation/
13-
artifacts/model_trainer/
14-
artifacts/model_evaluation/
15-
artifacts/feature_engineering/
9+
10+
# Keep DVC files but not the actual data/models
11+
!artifacts/.gitkeep
1612

1713
# Log files
1814
logs/

‎artifacts/data_validation/data_quality_report.json‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
{
2+
"train_data": {
3+
"shape": [
4+
10000,
5+
18
6+
],
7+
"missing_values": {
8+
"id": 0,
9+
"age": 0,
10+
"job": 0,
11+
"marital": 0,
12+
"education": 0,
13+
"default": 0,
14+
"balance": 0,
15+
"housing": 0,
16+
"loan": 0,
17+
"contact": 0,
18+
"day": 0,
19+
"month": 0,
20+
"duration": 0,
21+
"campaign": 0,
22+
"pdays": 0,
23+
"previous": 0,
24+
"poutcome": 0,
25+
"y": 0
26+
},
27+
"duplicates": 0,
28+
"numeric_columns": {
29+
"id": {
30+
"mean": 375395.7574,
31+
"std": 215829.53590458518,
32+
"min": 84.0,
33+
"max": 749992.0
34+
},
35+
"age": {
36+
"mean": 40.9016,
37+
"std": 10.111544974640493,
38+
"min": 19.0,
39+
"max": 93.0
40+
},
41+
"balance": {
42+
"mean": 1183.7881,
43+
"std": 2821.1241687944125,
44+
"min": -8019.0,
45+
"max": 88988.0
46+
},
47+
"day": {
48+
"mean": 16.148,
49+
"std": 8.265018272980724,
50+
"min": 1.0,
51+
"max": 31.0
52+
},
53+
"duration": {
54+
"mean": 256.2109,
55+
"std": 275.03809011897175,
56+
"min": 4.0,
57+
"max": 4785.0
58+
},
59+
"campaign": {
60+
"mean": 2.6401,
61+
"std": 2.8037756874990163,
62+
"min": 1.0,
63+
"max": 41.0
64+
},
65+
"pdays": {
66+
"mean": 20.6973,
67+
"std": 74.44830641285787,
68+
"min": -1.0,
69+
"max": 746.0
70+
},
71+
"previous": {
72+
"mean": 0.2977,
73+
"std": 1.349168905539098,
74+
"min": 0.0,
75+
"max": 32.0
76+
},
77+
"y": {
78+
"mean": 0.1207,
79+
"std": 0.32579460433288526,
80+
"min": 0.0,
81+
"max": 1.0
82+
}
83+
}
84+
},
85+
"test_data": {
86+
"shape": [
87+
10000,
88+
18
89+
],
90+
"missing_values": {
91+
"id": 0,
92+
"age": 0,
93+
"job": 0,
94+
"marital": 0,
95+
"education": 0,
96+
"default": 0,
97+
"balance": 0,
98+
"housing": 0,
99+
"loan": 0,
100+
"contact": 0,
101+
"day": 0,
102+
"month": 0,
103+
"duration": 0,
104+
"campaign": 0,
105+
"pdays": 0,
106+
"previous": 0,
107+
"poutcome": 0
108+
},
109+
"duplicates": 0,
110+
"numeric_columns": {
111+
"id": {
112+
"mean": 876579.5556,
113+
"std": 72313.41543772232,
114+
"min": 750041.0,
115+
"max": 999923.0
116+
},
117+
"age": {
118+
"mean": 41.0112,
119+
"std": 10.144606437704242,
120+
"min": 18.0,
121+
"max": 86.0
122+
},
123+
"balance": {
124+
"mean": 1184.9024,
125+
"std": 2481.8909831850406,
126+
"min": -6847.0,
127+
"max": 57435.0
128+
},
129+
"day": {
130+
"mean": 16.1672,
131+
"std": 8.34212686263143,
132+
"min": 1.0,
133+
"max": 31.0
134+
},
135+
"duration": {
136+
"mean": 259.1076,
137+
"std": 269.28706435967655,
138+
"min": 4.0,
139+
"max": 2420.0
140+
},
141+
"campaign": {
142+
"mean": 2.58,
143+
"std": 2.6714892367173704,
144+
"min": 1.0,
145+
"max": 35.0
146+
},
147+
"pdays": {
148+
"mean": 21.1596,
149+
"std": 73.87509146861206,
150+
"min": -1.0,
151+
"max": 524.0
152+
},
153+
"previous": {
154+
"mean": 0.3096,
155+
"std": 1.4347722009111423,
156+
"min": 0.0,
157+
"max": 23.0
158+
}
159+
}
160+
}
161+
}

‎artifacts/data_validation/drift_report.html‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
<html>
2+
<head>
3+
<title>Data Drift Report</title>
4+
<style>
5+
body { font-family: Arial, sans-serif; margin: 20px; }
6+
.header { background-color: #f4f4f4; padding: 20px; }
7+
.section { margin: 20px 0; }
8+
table { border-collapse: collapse; width: 100%; }
9+
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
10+
th { background-color: #f4f4f4; }
11+
.drift-detected { color: red; }
12+
.no-drift { color: green; }
13+
</style>
14+
</head>
15+
<body>
16+
<div class='header'>
17+
<h1>Data Drift Analysis Report</h1>
18+
<p>Validation Status: <strong>Passed</strong></p>
19+
</div>
20+
<div class='section'>
21+
<h2>Validation Details</h2>
22+
<h3>Training Data Validation</h3>
23+
<p>Status: True</p>
24+
<p>Missing Columns: None</p>
25+
<p>Data Type Errors: None</p>
26+
<h3>Test Data Validation</h3>
27+
<p>Status: True</p>
28+
<p>Missing Columns: None</p>
29+
</div>
30+
<div class='section'>
31+
<h2>Data Drift Analysis</h2>
32+
<table>
33+
<tr>
34+
<th>Feature</th>
35+
<th>KS Statistic</th>
36+
<th>P-Value</th>
37+
<th>Drift Status</th>
38+
</tr>
39+
<tr>
40+
<td>day</td>
41+
<td>0.0085</td>
42+
<td>0.9986</td>
43+
<td class='no-drift'>No Drift</td>
44+
</tr>
45+
<tr>
46+
<td>balance</td>
47+
<td>0.0185</td>
48+
<td>0.4969</td>
49+
<td class='no-drift'>No Drift</td>
50+
</tr>
51+
<tr>
52+
<td>duration</td>
53+
<td>0.0170</td>
54+
<td>0.6062</td>
55+
<td class='no-drift'>No Drift</td>
56+
</tr>
57+
<tr>
58+
<td>age</td>
59+
<td>0.0145</td>
60+
<td>0.7910</td>
61+
<td class='no-drift'>No Drift</td>
62+
</tr>
63+
<tr>
64+
<td>campaign</td>
65+
<td>0.0181</td>
66+
<td>0.5253</td>
67+
<td class='no-drift'>No Drift</td>
68+
</tr>
69+
<tr>
70+
<td>previous</td>
71+
<td>0.0038</td>
72+
<td>1.0000</td>
73+
<td class='no-drift'>No Drift</td>
74+
</tr>
75+
<tr>
76+
<td>pdays</td>
77+
<td>0.0066</td>
78+
<td>1.0000</td>
79+
<td class='no-drift'>No Drift</td>
80+
</tr>
81+
<tr>
82+
<td>id</td>
83+
<td>1.0000</td>
84+
<td>0.0000</td>
85+
<td class='drift-detected'>Drift Detected</td>
86+
</tr>
87+
</table>
88+
</div>
89+
</body>
90+
</html>

‎artifacts/data_validation/validation_status.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"validation_status": true,
3+
"train_validation": {
4+
"status": true,
5+
"missing_columns": [],
6+
"dtype_errors": []
7+
},
8+
"test_validation": {
9+
"status": true,
10+
"missing_columns": []
11+
}
12+
}

‎artifacts/model_trainer/model.pkl‎

277 KB
Binary file not shown.

0 commit comments

Comments
 (0)