Open dedenbangkit opened 1 year ago
wai-sdg-portal-worker-1 | Filename: ./tasks/validation.py wai-sdg-portal-worker-1 | wai-sdg-portal-worker-1 | Line # Mem usage Increment Occurrences Line Contents wai-sdg-portal-worker-1 | ============================================================= wai-sdg-portal-worker-1 | 243 115.7 MiB 115.7 MiB 1 @profile wai-sdg-portal-worker-1 | 244 def validate(session: Session, form: int, administration: int, file: str): wai-sdg-portal-worker-1 | 245 115.7 MiB 0.0 MiB 1 try: wai-sdg-portal-worker-1 | 246 124.1 MiB 8.4 MiB 1 sheet_names = validate_sheet_name(file) wai-sdg-portal-worker-1 | 247 124.1 MiB 0.0 MiB 1 template_sheets = ["data", "definitions", "administration"] wai-sdg-portal-worker-1 | 248 124.1 MiB 0.0 MiB 1 TESTING = os.environ.get("TESTING") wai-sdg-portal-worker-1 | 249 124.1 MiB 0.0 MiB 1 if TESTING: wai-sdg-portal-worker-1 | 250 template_sheets = ["data"] wai-sdg-portal-worker-1 | 251 124.1 MiB 0.0 MiB 4 for sheet_tab in template_sheets: wai-sdg-portal-worker-1 | 252 124.1 MiB 0.0 MiB 3 if sheet_tab not in sheet_names: wai-sdg-portal-worker-1 | 253 return [ wai-sdg-portal-worker-1 | 254 { wai-sdg-portal-worker-1 | 255 "error": ExcelError.sheet, wai-sdg-portal-worker-1 | 256 "error_message": ValidationText.template_validation.value, wai-sdg-portal-worker-1 | 257 "sheets": ",".join(sheet_names), wai-sdg-portal-worker-1 | 258 } wai-sdg-portal-worker-1 | 259 ] wai-sdg-portal-worker-1 | 260 124.1 MiB 0.0 MiB 1 questions = crud_question.get_excel_question(session=session, form=form) wai-sdg-portal-worker-1 | 261 124.3 MiB 0.2 MiB 23 header_names = [q.to_excel_header for q in questions.all()] wai-sdg-portal-worker-1 | 262 126.0 MiB 1.6 MiB 1 df = pd.read_excel(file, sheet_name="data") wai-sdg-portal-worker-1 | 263 126.0 MiB 0.0 MiB 1 if df.shape[0] == 0: wai-sdg-portal-worker-1 | 264 return [ wai-sdg-portal-worker-1 | 265 { wai-sdg-portal-worker-1 | 266 "error": ExcelError.sheet, wai-sdg-portal-worker-1 | 267 "error_message": ValidationText.file_empty_validation.value, wai-sdg-portal-worker-1 | 268 } wai-sdg-portal-worker-1 | 269 ] wai-sdg-portal-worker-1 | 270 126.0 MiB 0.0 MiB 1 excel_head = {} wai-sdg-portal-worker-1 | 271 126.0 MiB 0.0 MiB 1 excel_cols = list(itertools.islice(generate_excel_columns(), df.shape[1])) wai-sdg-portal-worker-1 | 272 126.0 MiB 0.0 MiB 21 for index, header in enumerate(list(df)): wai-sdg-portal-worker-1 | 273 126.0 MiB 0.0 MiB 20 excel_head.update({excel_cols[index]: header}) wai-sdg-portal-worker-1 | 274 126.0 MiB 0.0 MiB 1 header_error = [] wai-sdg-portal-worker-1 | 275 126.0 MiB 0.0 MiB 1 data_error = [] wai-sdg-portal-worker-1 | 276 126.0 MiB 0.0 MiB 2 childs = crud_administration.get_all_childs( wai-sdg-portal-worker-1 | 277 126.0 MiB 0.0 MiB 1 session=session, parents=[administration], current=[] wai-sdg-portal-worker-1 | 278 ) wai-sdg-portal-worker-1 | 279 126.0 MiB 0.0 MiB 2 adm = crud_administration.get_administration_by_id( wai-sdg-portal-worker-1 | 280 126.0 MiB 0.0 MiB 1 session=session, id=administration wai-sdg-portal-worker-1 | 281 ) wai-sdg-portal-worker-1 | 282 126.0 MiB 0.0 MiB 1 adm = {"id": adm.id, "name": adm.name, "childs": childs} wai-sdg-portal-worker-1 | 283 126.0 MiB 0.0 MiB 1 answered = [] wai-sdg-portal-worker-1 | 284 131.3 MiB 0.0 MiB 21 for col in excel_head: wai-sdg-portal-worker-1 | 285 131.0 MiB 0.0 MiB 20 header = excel_head[col] wai-sdg-portal-worker-1 | 286 131.0 MiB 0.0 MiB 20 error = validate_header_names(header, f"{col}1", header_names) wai-sdg-portal-worker-1 | 287 131.0 MiB 0.0 MiB 20 if error: wai-sdg-portal-worker-1 | 288 header_error.append(error) wai-sdg-portal-worker-1 | 289 131.0 MiB 0.0 MiB 20 if not error: wai-sdg-portal-worker-1 | 290 131.0 MiB 0.0 MiB 20 qid = header.split("|")[0] wai-sdg-portal-worker-1 | 291 131.0 MiB 0.0 MiB 20 question = questions.filter(Question.id == int(qid)).first() wai-sdg-portal-worker-1 | 292 131.0 MiB 0.0 MiB 20 answers = list(df[header]) wai-sdg-portal-worker-1 | 293 131.3 MiB 0.0 MiB 17820 for i, answer in enumerate(answers): wai-sdg-portal-worker-1 | 294 131.3 MiB 0.0 MiB 17800 ix = i + 2 wai-sdg-portal-worker-1 | 295 131.3 MiB 0.5 MiB 17800 valid_deps = False wai-sdg-portal-worker-1 | 296 131.3 MiB 0.0 MiB 17800 answer_deps = None wai-sdg-portal-worker-1 | 297 131.3 MiB 0.0 MiB 35600 answered.append( wai-sdg-portal-worker-1 | 298 131.3 MiB 0.5 MiB 17800 { wai-sdg-portal-worker-1 | 299 131.3 MiB 0.0 MiB 17800 "id": question.id, wai-sdg-portal-worker-1 | 300 131.3 MiB 0.0 MiB 17800 "answer": answer, wai-sdg-portal-worker-1 | 301 131.3 MiB 0.2 MiB 17800 "cell": f"{col}{ix}", wai-sdg-portal-worker-1 | 302 131.3 MiB 0.0 MiB 17800 "index": ix, wai-sdg-portal-worker-1 | 303 } wai-sdg-portal-worker-1 | 304 ) wai-sdg-portal-worker-1 | 305 131.3 MiB 0.0 MiB 17800 if question.dependency: wai-sdg-portal-worker-1 | 306 131.3 MiB 0.2 MiB 19580 valid_deps, answer_deps = dependency_checker( wai-sdg-portal-worker-1 | 307 131.3 MiB 0.0 MiB 9790 qs=question.dependency, answered=answered, index=ix wai-sdg-portal-worker-1 | 308 ) wai-sdg-portal-worker-1 | 309 131.3 MiB 3.1 MiB 35600 error = validate_row_data( wai-sdg-portal-worker-1 | 310 131.3 MiB 0.0 MiB 17800 session, wai-sdg-portal-worker-1 | 311 131.3 MiB 0.8 MiB 17800 f"{col}{ix}", wai-sdg-portal-worker-1 | 312 131.3 MiB 0.0 MiB 17800 answer, wai-sdg-portal-worker-1 | 313 131.3 MiB 0.0 MiB 17800 question, wai-sdg-portal-worker-1 | 314 131.3 MiB 0.0 MiB 17800 adm, wai-sdg-portal-worker-1 | 315 131.3 MiB 0.0 MiB 17800 valid_deps, wai-sdg-portal-worker-1 | 316 131.3 MiB 0.0 MiB 17800 answer_deps, wai-sdg-portal-worker-1 | 317 ) wai-sdg-portal-worker-1 | 318 131.3 MiB 0.0 MiB 17800 if error: wai-sdg-portal-worker-1 | 319 data_error.append(error) wai-sdg-portal-worker-1 | 320 131.3 MiB 0.0 MiB 1 return header_error + data_error wai-sdg-portal-worker-1 | 321 except Exception as e: wai-sdg-portal-worker-1 | 322 print("VALIDATION ERROR", str(e)) wai-sdg-portal-worker-1 | 323 return None
dedenbangkit
commented
1 year ago dedenbangkit
commented
1 year ago