Add try clause to DataFusionHook.wait_for_pipeline_state (#10031)

turbaszek · web-flow · commit 27020f8e5885 · 2020-08-03T10:54:09.000+02:00
Sometimes it may happen that the pipeline is not visible instantly in DataFusion so retrieving it will result in 404 closes: #10030
diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/airflow/providers/google/cloud/hooks/datafusion.py
@@ -112,13 +112,15 @@ def wait_for_pipeline_state(
         start_time = monotonic()
         current_state = None
         while monotonic() - start_time < timeout:
-            current_state = self._get_workflow_state(
-                pipeline_name=pipeline_name,
-                pipeline_id=pipeline_id,
-                instance_url=instance_url,
-                namespace=namespace,
-            )
-
+            try:
+                current_state = self._get_workflow_state(
+                    pipeline_name=pipeline_name,
+                    pipeline_id=pipeline_id,
+                    instance_url=instance_url,
+                    namespace=namespace,
+                )
+            except AirflowException:
+                pass  # Because the pipeline may not be visible in system yet
             if current_state in success_states:
                 return
             if current_state in failure_states:
