Add image annotate example

examples/image_annotate/definitions/all_imagesets.sqlx

@@ -0,0 +1,10 @@
+config {
+    type: "operations",
+    hasOutput: true
+}
+
+  -- Create the object table for the images
+CREATE EXTERNAL TABLE IF NOT EXISTS
+  ${self()}
+WITH CONNECTION `us.gcs` OPTIONS( object_metadata = 'SIMPLE',
+    uris = ['${constants.IMAGE_BUCKET}'] );

examples/image_annotate/definitions/annotated_imagesets.sqlx

@@ -0,0 +1,25 @@
+config {
+    type: "incremental"
+}
+
+SELECT
+  *
+FROM
+  ${ref("object_table")}
+WHERE
+  ml_annotate_image_status NOT LIKE "${constants.RETRYABLE_ERROR}"
+  ${
+      when(incremental(), `AND
+      ${constants.MERGE_COLUMN} NOT IN (
+      SELECT
+        ${constants.MERGE_COLUMN}
+      FROM
+        ${self()})
+      OR updated > (
+      SELECT
+        MAX(updated)
+      FROM
+        ${self()})`)
+  }
+LIMIT
+  ${constants.BATCH_SIZE}

examples/image_annotate/definitions/annotated_imagesets_errors.sqlx

@@ -0,0 +1,10 @@
+config {
+    type: "assertion"
+}
+
+SELECT
+  *
+FROM
+  ${ref("object_table")}
+WHERE
+  ml_annotate_image_status LIKE "${constants.RETRYABLE_ERROR}"

examples/image_annotate/definitions/object_table.sqlx

@@ -0,0 +1,10 @@
+config {
+    type: "table",
+}
+
+SELECT
+  *
+FROM
+  ML.ANNOTATE_IMAGE( MODEL ${ref("vision_model")},
+    TABLE ${ref("unannotated_imagesets")},
+    STRUCT(['LABEL_DETECTION'] AS vision_features))

examples/image_annotate/definitions/unannotated_imagesets.sqlx

@@ -0,0 +1,19 @@
+config {
+    type: "view"
+}
+
+js {
+    // Ref is avoided here to avoid the circular dependency chain.
+    const ANNOTATED_IMAGESETS = `${ctx.schema()}` + ".annotated_imagesets"
+}
+
+SELECT
+  *
+FROM
+  ${ref("all_imagesets")}
+LEFT JOIN
+  `${ANNOTATED_IMAGESETS}`
+ON
+  ${ref("all_imagesets")}.uri = `${ANNOTATED_IMAGESETS}`.uri
+WHERE
+  `${ANNOTATED_IMAGESETS}`.uri IS NULL

examples/image_annotate/definitions/vision_model.sqlx

@@ -0,0 +1,9 @@
+config {
+    type: "operations",
+    hasOutput: true
+}
+
+  -- Model for connecting to the Vision API
+CREATE MODEL IF NOT EXISTS
+  ${self()} REMOTE
+WITH CONNECTION `us.gcs` OPTIONS (remote_service_type = 'cloud_ai_vision_v1');

examples/image_annotate/includes/constants.js

@@ -0,0 +1,6 @@
+module.exports = {
+  BATCH_SIZE: 100,
+  MERGE_COLUMN: "uri",
+  RETRYABLE_ERROR: "A retryable error occurred:",
+  IMAGE_BUCKET: "gs://example-bucket/imagesets/*"
+};

examples/image_annotate/readme.md

@@ -0,0 +1,9 @@
+# Image Annotate
+
+Use BigQuery ML to annotate images from a GCS bucket.
+
+Images where annotation failed are filtered out, and retried in subsequent runs.
+
+![Image Annotation DAG](./image-annotation-dag.png?raw=true "Image Annotation DAG")
+
+<!-- TODO(ekrekr): add more info on setting up BigQuery connections. -->

examples/image_annotate/workflow_settings.yaml

@@ -0,0 +1,4 @@
+defaultProject: dataform-demos
+defaultLocation: US
+defaultDataset: sample
+defaultAssertionDataset: sample_assertions