Fix Spark module distribution: properly zip and distribute impc_etl to workers

Author: ficolo

impc_etl/utils/spark.py

@@ -96,6 +96,7 @@ def wrapper():
             )
 
             # Create a temporary zip file of the impc_etl module
+            temp_zip_path = None
             temp_zip_fd, temp_zip_path = tempfile.mkstemp(suffix='.zip', prefix='impc_etl_')
             os.close(temp_zip_fd)  # Close the file descriptor
 
@@ -123,7 +124,7 @@ def wrapper():
                 task_logger.info(f"Added impc_etl module to Spark workers")
             except Exception as e:
                 task_logger.error(f"Failed to prepare impc_etl module: {e}")
-                if os.path.exists(temp_zip_path):
+                if temp_zip_path and os.path.exists(temp_zip_path):
                     os.unlink(temp_zip_path)
                 raise
 
@@ -146,7 +147,7 @@ def wrapper():
                     # Continue execution as the task may have completed successfully
                 finally:
                     # Clean up temporary zip file
-                    if 'temp_zip_path' in locals() and os.path.exists(temp_zip_path):
+                    if temp_zip_path and os.path.exists(temp_zip_path):
                         try:
                             os.unlink(temp_zip_path)
                             task_logger.info("Cleaned up temporary impc_etl zip file")