/*

* Copyright 2024 Google LLC

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

* http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

config {

type: "incremental",

// tags: ["summary", "daily"],

uniqueKey: audit_logs.adminActionsUniqueColumns,

bigquery: {

partitionBy: "day",

updatePartitionFilter:

"day >= time_window_checkpoint"

}

}

pre_operations {

DECLARE time_window_checkpoint DEFAULT (

${when(incremental(),

`SELECT MAX(day) FROM ${self()}`,

`SELECT DATE("2000-01-01")`)}

);

}

SELECT

* EXCEPT (${

functions.embeddingModelOutputColumns(

dataform.projectConfig.vars.text_embedding_model

).join(',')

}),

ml_generate_embedding_result AS text_embedding

FROM ML.GENERATE_EMBEDDING(

MODEL ${ref("text_embedding")},

(

SELECT *

FROM ${ref("actions_summary_daily")}

WHERE

${ when(incremental(), `day >= time_window_checkpoint AND`) }

day >= DATE_SUB(CURRENT_DATE(), INTERVAL ${dataform.projectConfig.vars.summary_lookback_days} DAY)

),

STRUCT(TRUE AS flatten_json_output)

)

-- Create vector index over text_embedding column

-- Do nothing if vector index exists already (during incremental update)

post_operations {

CREATE VECTOR INDEX IF NOT EXISTS `my_vector_index`

ON ${self()}(text_embedding)

OPTIONS(distance_type='COSINE', index_type='IVF');

}