Coverage for python/pyairflowtester/rules/__init__.py: 86%

1""" 

2Rules package for PyAirflowTester. 

3 

4Contains rule definitions for DAG, dbt, configuration, and other analysis. 

5""" 

6 

7from pyairflowtester.rules.config import ( 

8 AirflowCfgPoolConfigurationRule, 

9 ConcurrencyConfigurationRule, 

10 DAGFolderConfigurationRule, 

11 DatabaseBackupRule, 

12 EncryptionConfigurationRule, 

13 ExecutorConfigurationRule, 

14 LogRetentionRule, 

15 LogStorageRule, 

16 MaxActiveRunsRule, 

17 QueueConfigurationRule, 

18 RBACConfigurationRule, 

19 SchedulerConfigurationRule, 

20 TLSConfigurationRule, 

21 WorkerConfigurationRule, 

22 XComConfigurationRule, 

23) 

24from pyairflowtester.rules.dag import ( 

25 CircularDependencyRule, 

26 ExpensiveImportsRule, 

27 MissingSLARule, 

28 ParseTimeRule, 

29) 

30from pyairflowtester.rules.dag_advanced import ( 

31 AlertingConfigurationRule, 

32 BranchComplexityRule, 

33 CatchupConfigRule, 

34 DocumentationRule, 

35 HardcodedConnectionRule, 

36 OperatorDeprecationRule, 

37 RetryConfigurationRule, 

38 SecretsInCodeRule, 

39 SensorTimeoutRule, 

40 SourceCodePoolConfigurationRule, 

41 TaskCountRule, 

42) 

43from pyairflowtester.rules.dbt import ( 

44 MissingTestsRule, 

45 RedundantTestsRule, 

46 UntestedModelRule, 

47) 

48 

49 

50def get_all_rules(): 

51 """Get all available rules (35 total).""" 

52 return [ 

53 # DAG Analysis Rules (AFW001-AFW015) 

54 { 

55 "id": "AFW001", 

56 "name": "Circular Dependency", 

57 "severity": "critical", 

58 "category": "reliability", 

59 "execution_mode": "static", 

60 "description": "Detect circular task dependencies", 

61 }, 

62 { 

63 "id": "AFW002", 

64 "name": "Missing SLA", 

65 "severity": "high", 

66 "category": "reliability", 

67 "execution_mode": "static", 

68 "description": "Detect missing SLA on production DAG", 

69 }, 

70 { 

71 "id": "AFW003", 

72 "name": "Expensive Imports", 

73 "severity": "medium", 

74 "category": "performance", 

75 "execution_mode": "static", 

76 "description": "Detect expensive imports in DAG file", 

77 }, 

78 { 

79 "id": "AFW004", 

80 "name": "Parse Time", 

81 "severity": "medium", 

82 "category": "performance", 

83 "execution_mode": "static", 

84 "description": "Analyze DAG parse time", 

85 }, 

86 { 

87 "id": "AFW005", 

88 "name": "Excessive Task Count", 

89 "severity": "medium", 

90 "category": "performance", 

91 "execution_mode": "static", 

92 "description": "Detect DAGs with > 500 tasks", 

93 }, 

94 { 

95 "id": "AFW006", 

96 "name": "Risky Catchup Configuration", 

97 "severity": "high", 

98 "category": "reliability", 

99 "execution_mode": "static", 

100 "description": "Detect catchup=True without backfill strategy", 

101 }, 

102 { 

103 "id": "AFW007", 

104 "name": "Default Pool Usage", 

105 "severity": "medium", 

106 "category": "reliability", 

107 "execution_mode": "static", 

108 "description": "Detect tasks using default pool", 

109 }, 

110 { 

111 "id": "AFW008", 

112 "name": "Hardcoded Connection ID", 

113 "severity": "high", 

114 "category": "maintainability", 

115 "execution_mode": "static", 

116 "description": "Detect hardcoded connection strings", 

117 }, 

118 { 

119 "id": "AFW009", 

120 "name": "Secrets in Code", 

121 "severity": "critical", 

122 "category": "security", 

123 "execution_mode": "static", 

124 "description": "Detect hardcoded secrets", 

125 }, 

126 { 

127 "id": "AFW010", 

128 "name": "Excessive Retries", 

129 "severity": "medium", 

130 "category": "reliability", 

131 "execution_mode": "static", 

132 "description": "Detect retries > 5 without backoff", 

133 }, 

134 { 

135 "id": "AFW011", 

136 "name": "Sensor Timeout Risk", 

137 "severity": "medium", 

138 "category": "reliability", 

139 "execution_mode": "static", 

140 "description": "Detect sensors with > 1h timeout", 

141 }, 

142 { 

143 "id": "AFW012", 

144 "name": "Complex Branching", 

145 "severity": "low", 

146 "category": "maintainability", 

147 "execution_mode": "static", 

148 "description": "Detect complex BranchPythonOperator logic", 

149 }, 

150 { 

151 "id": "AFW013", 

152 "name": "Missing Documentation", 

153 "severity": "low", 

154 "category": "maintainability", 

155 "execution_mode": "static", 

156 "description": "Detect DAGs without description", 

157 }, 

158 { 

159 "id": "AFW014", 

160 "name": "No Alerting Configured", 

161 "severity": "medium", 

162 "category": "reliability", 

163 "execution_mode": "static", 

164 "description": "Detect production DAGs without alerts", 

165 }, 

166 { 

167 "id": "AFW015", 

168 "name": "Deprecated Operator", 

169 "severity": "medium", 

170 "category": "maintainability", 

171 "execution_mode": "static", 

172 "description": "Detect deprecated Airflow operators", 

173 }, 

174 # dbt Analysis Rules (DBT001-DBT003) 

175 { 

176 "id": "DBT001", 

177 "name": "Missing Tests", 

178 "severity": "high", 

179 "category": "data_quality", 

180 "execution_mode": "static", 

181 "description": "Detect models without tests", 

182 }, 

183 { 

184 "id": "DBT002", 

185 "name": "Redundant Tests", 

186 "severity": "low", 

187 "category": "maintainability", 

188 "execution_mode": "static", 

189 "description": "Detect redundant test definitions", 

190 }, 

191 { 

192 "id": "DBT003", 

193 "name": "Untested Model", 

194 "severity": "medium", 

195 "category": "data_quality", 

196 "execution_mode": "static", 

197 "description": "Detect untested public models", 

198 }, 

199 # Configuration Audit Rules (CFG001-CFG015) 

200 { 

201 "id": "CFG001", 

202 "name": "Executor Misconfiguration", 

203 "severity": "high", 

204 "category": "performance", 

205 "execution_mode": "static", 

206 "description": "Check executor type matches workload", 

207 }, 

208 { 

209 "id": "CFG002", 

210 "name": "Pool Size Mismatch", 

211 "severity": "medium", 

212 "category": "reliability", 

213 "execution_mode": "static", 

214 "description": "Check pool sizes adequate", 

215 }, 

216 { 

217 "id": "CFG003", 

218 "name": "Concurrency Mismatch", 

219 "severity": "medium", 

220 "category": "performance", 

221 "execution_mode": "static", 

222 "description": "Check DAG/task concurrency settings", 

223 }, 

224 { 

225 "id": "CFG004", 

226 "name": "Queue Bottleneck", 

227 "severity": "medium", 

228 "category": "reliability", 

229 "execution_mode": "static", 

230 "description": "Check queue configuration", 

231 }, 

232 { 

233 "id": "CFG005", 

234 "name": "Max Active Runs Not Limited", 

235 "severity": "high", 

236 "category": "reliability", 

237 "execution_mode": "static", 

238 "description": "Check max_active_dag_runs limit", 

239 }, 

240 { 

241 "id": "CFG006", 

242 "name": "XCom Backend Issue", 

243 "severity": "medium", 

244 "category": "performance", 

245 "execution_mode": "static", 

246 "description": "Check XCom configuration", 

247 }, 

248 { 

249 "id": "CFG007", 

250 "name": "Insufficient Log Retention", 

251 "severity": "medium", 

252 "category": "compliance", 

253 "execution_mode": "static", 

254 "description": "Check log retention >= 30 days", 

255 }, 

256 { 

257 "id": "CFG008", 

258 "name": "Encryption Not Configured", 

259 "severity": "high", 

260 "category": "security", 

261 "execution_mode": "static", 

262 "description": "Check Fernet encryption configured", 

263 }, 

264 { 

265 "id": "CFG009", 

266 "name": "TLS Not Enabled", 

267 "severity": "high", 

268 "category": "security", 

269 "execution_mode": "static", 

270 "description": "Check TLS for web UI", 

271 }, 

272 { 

273 "id": "CFG010", 

274 "name": "RBAC Disabled", 

275 "severity": "high", 

276 "category": "security", 

277 "execution_mode": "static", 

278 "description": "Check RBAC enabled", 

279 }, 

280 { 

281 "id": "CFG011", 

282 "name": "Scheduler Misconfiguration", 

283 "severity": "medium", 

284 "category": "performance", 

285 "execution_mode": "static", 

286 "description": "Check scheduler settings", 

287 }, 

288 { 

289 "id": "CFG012", 

290 "name": "Worker Misconfiguration", 

291 "severity": "medium", 

292 "category": "reliability", 

293 "execution_mode": "static", 

294 "description": "Check worker configuration", 

295 }, 

296 { 

297 "id": "CFG013", 

298 "name": "Local Log Storage", 

299 "severity": "high", 

300 "category": "reliability", 

301 "execution_mode": "static", 

302 "description": "Check log storage location", 

303 }, 

304 { 

305 "id": "CFG014", 

306 "name": "No Database Backup", 

307 "severity": "high", 

308 "category": "reliability", 

309 "execution_mode": "static", 

310 "description": "Check database backup configuration", 

311 }, 

312 { 

313 "id": "CFG015", 

314 "name": "DAG Folder On NFS", 

315 "severity": "medium", 

316 "category": "performance", 

317 "execution_mode": "static", 

318 "description": "Check DAG folder location", 

319 }, 

320 ] 

321 

322 

323__all__ = [ 

324 # DAG Rules 

325 "CircularDependencyRule", 

326 "MissingSLARule", 

327 "ExpensiveImportsRule", 

328 "ParseTimeRule", 

329 "TaskCountRule", 

330 "CatchupConfigRule", 

331 "SourceCodePoolConfigurationRule", 

332 "HardcodedConnectionRule", 

333 "SecretsInCodeRule", 

334 "RetryConfigurationRule", 

335 "SensorTimeoutRule", 

336 "BranchComplexityRule", 

337 "DocumentationRule", 

338 "AlertingConfigurationRule", 

339 "OperatorDeprecationRule", 

340 # dbt Rules 

341 "MissingTestsRule", 

342 "RedundantTestsRule", 

343 "UntestedModelRule", 

344 # Config Rules 

345 "ExecutorConfigurationRule", 

346 "AirflowCfgPoolConfigurationRule", 

347 "ConcurrencyConfigurationRule", 

348 "QueueConfigurationRule", 

349 "MaxActiveRunsRule", 

350 "XComConfigurationRule", 

351 "LogRetentionRule", 

352 "EncryptionConfigurationRule", 

353 "TLSConfigurationRule", 

354 "RBACConfigurationRule", 

355 "SchedulerConfigurationRule", 

356 "WorkerConfigurationRule", 

357 "LogStorageRule", 

358 "DatabaseBackupRule", 

359 "DAGFolderConfigurationRule", 

360 "get_all_rules", 

361]