I want to create a master view combining all the tables passed as input to Snowflake Stored procedure. Please help on how the code can be framed for this.
create or replace procedure TEST_PROC("SRC_DB" VARCHAR(30),
"SRC_SCHEMA" VARCHAR(30), "TGT_DB" VARCHAR(30), "TGT_SCHEMA"
VARCHAR(30))
RETURNS varchar
LANGUAGE JAVASCRIPT
EXECUTE AS OWNER
as
$$
var result = '';
var tab = 'TABLE1,TABLE2'
var get_tables = `
with cte as(select value from table(SPLIT_TO_TABLE
(('${tab}'),','))
) select value from cte;`
var tables_name_master=snowflake.execute ({sqlText: get_tables});
var lcols_agg = '';
while(tables_name_master.next()){
var table_value = tables_name_master.getColumnValue(1);
var column_list = `
WITH cte2 as (select COLUMN_NAME , listagg(TABLE_NAME, ', ')
within group (order by COLUMN_NAME) A
from ${SRC_DB}.information_schema.COLUMNS
where TABLE_SCHEMA= '${SRC_SCHEMA}' and TABLE_NAME in (select
value from table(SPLIT_TO_TABLE (('${tab}'),',')))
group by COLUMN_NAME order by COLUMN_NAME
),
cte3 as (select 'x' x, COLUMN_NAME,iff(contains(A,'${table_value}'),COLUMN_NAME,CONCAT('NULL AS \"',COLUMN_NAME,'\"')) valuess from cte2 order by COLUMN_NAME
)select listagg(valuess,',') final FROM cte3 GROUP BY x
`;
var rs = snowflake.execute({ sqlText: column_list });
while(rs.next()){
lcols_agg += "SELECT " + rs.getColumnValue(1) + " FROM "+ SRC_DB+"."+SRC_SCHEMA+"."+tables_name_master.getColumnValue(1) + "\n" +"UNION " +"\n"
}
}
var count1 = 0 ;
count1 = lcols_agg.length
result = lcols_agg.substring(0,(count1-7));
const create_union_view = `
create or replace view abcd AS ${result}
;`
var view_create = snowflake.execute({ sqlText: create_union_view });
view_create.next()
return result
$$
;
call SP_TEST('ABC','DEF','PQR','STU');
THis generates my final view statement as
CREATE OR REPLACE VIEW ABCD AS
COL1,COL2,COL3
UNION
NULL AS "COL2",NULL AS "COL3",COL1
Now due to mismatch of order od columns in union the view is throwing error while we do select * from abcd, any way we can have the columns of both tables in same order or any other work around?
You need to ORDER BY the function listagg after the cte3 expression:
instead of
select listagg(valuess,',') final FROM cte3 GROUP BY x
change it to:
select listagg(valuess,',') within group (order by valuess) final FROM cte3 GROUP BY x
See if that helps to resolve the order issue.
I am executing
CALL LOAD_CUSTDIM_HOUSTONS:
This does the below:
I am running this code to get unique Customer Details such as Consumer No and Customer Name from STG_HOUSTON table and Get Max(CUST_DImKEY) from the DIM_CUSTOMER Key for the CUSTOMER Name from above i.e Houston's.
Next I am checking if the Max key value is null i mean no match for the CUSTOMER so Key value first time to be always 1 next time it will be max value + 1.
Once we get this bind this value and the select where clause will get the unique customer and insert into Customer Dimension table.
Stored procedure looks like this:
create or replace procedure LOAD_CUSTDIM_HOUSTONS()
returns varchar not null
language javascript
as
$$
var return_value = "";
try
{
var SQL_SMT = "SELECT DISTINCT CONSUMER_NUMBER,CUSTOMER_NAME FROM STG_HOUSTONS ORDER BY CONSUMER_NUMBER ASC";
var stmt1 = snowflake.createStatement(
{
sqlText: SQL_SMT
}
);
/* Creates result set */
var res1 = stmt1.execute();
while (res1.next())
{
var var_ConsumNo=res1.getColumnValue(1);
var var_custname=res1.getColumnValue(2);
return_value =" Inside While";
var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
var res2 = t_stmt.execute();
res2.next();
var_Custkey =res2.getColumnValue(1)
return_value= return_value + var_Custkey;
if (var_CustKey == null)
{
var_CustdimKey =1
}
else
{
var_CustdimKey=var_CustKey+1
}
return_value= return_value + var_CustdimKey.toString();
var SQL_INSDIMCUS="INSERT INTO DIM_CUSTOMER(CUSTOMER_KEY,CUSTOMER_ID,CUSTOMER_NAME,CONSUMER_ID,CONSUMER_NAME,CUSTOMER_IND,TODATE)" + " SELECT DISTINCT :1, CUSTOMER_NUMBER,CUSTOMER_NAME,CONSUMER_NUMBER,CONSUMER_SHPPNG_FNAME + ' ' + CONSUMER_SHPPNG_LNAME AS CONSUMER_NAME, FALSE, NULL FROM STG_HOUSTONS WHERE CONSUMER_NUMBER = :2 ;";
var stmt = snowflake.createStatement({sqlText: SQL_INSDIMCUS,binds: [var_CustdimKey,var_ConsumNo]} );
var res3 = stmt.execute();
result = "COMPLETED SUCCESSFULLY!";
}
}
catch (err)
{ result = "Failed!";
var time_st = snowflake.execute( {sqlText: `SELECT CURRENT_TIMESTAMP;`} );
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
});
}
return return_value;
$$
;
This is the function I am using it to get max(DIM CUST KEY).
create or replace function GET_CUSTDIMKEYS ( CUSTOMERNAME varchar )
returns bigint
as 'Select max(CUSTOMER_KEY)
from PUBLIC.DIM_CUSTOMER
where CUSTOMER_NAME = CUSTOMERNAME ';
I tried with Binding variable / direct substitution of the values to dynamically create the SQL and execute it.
Using DBeaver21.1.1 IDE to create snowflake stored procedure and functions
This is the execution log Created by the IDE.
2021-10-05 20:44:48.030 - SQL Error [100183] [P0000]: Execution error in store procedure LOAD_CUSTDIM_HOUSTONS:
Unsupported type for binding argument 2undefined
At Snowflake.execute, line 49 position 14
org.jkiss.dbeaver.model.sql.DBSQLException: SQL Error [100183] [P0000]: Execution error in store procedure LOAD_CUSTDIM_HOUSTONS:
Unsupported type for binding argument 2undefined
At Snowflake.execute, line 49 position 14
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCPreparedStatementImpl.executeStatement(JDBCPreparedStatementImpl.java:208)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeStatement(SQLQueryJob.java:510)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.lambda$0(SQLQueryJob.java:441)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:171)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeSingleQuery(SQLQueryJob.java:428)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.extractData(SQLQueryJob.java:813)
at org.jkiss.dbeaver.ui.editors.sql.SQLEditor$QueryResultsContainer.readData(SQLEditor.java:3280)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.lambda$0(ResultSetJobDataRead.java:118)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:171)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.run(ResultSetJobDataRead.java:116)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetViewer$ResultSetDataPumpJob.run(ResultSetViewer.java:4624)
at org.jkiss.dbeaver.model.runtime.AbstractJob.run(AbstractJob.java:105)
at org.eclipse.core.internal.jobs.Worker.run(Worker.java:63)
Line 49 is:
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
The error message:
Unsupported type for binding argument 2undefined
means the variable passed into the binding argument is not defined.
This tells me that the variable "err" was not initialized properly. I have seen some cases that under certain conditions, like when calling a function that does not exist on an object, though exception is thrown, the argument passed into the catch block is still NULL.
You need to look into the query history for this particular session, and find out which one of the queries that ran from the SP failed, and find out why.
And in the catch block, try to check the "err" object before using its attributes, something like:
if (err) {
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
});
} else {
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS','my code', 'my state','my error', 'my stack']
});
}
This can avoid such failure.
This is in addition to Eric's answer. You should not wrap strings in single quotes when used as bind variables:
var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
This should be:
//var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
If you wrap a string type bind variable in single quotes, the single quotes become part of the variable. The practical effect is that instead of running a statement like this:
SELECT GET_CUSTDIMKEYS('MyCustomer'); -- Looks for MyCustomer
You run a statement like this:
SELECT GET_CUSTDIMKEYS('\'MyCustomer\''); -- Looks for 'MyCustomer'
The issue is first the second result set should have a while loop which was missing,
And rest of the code proceeds will solve the issue and also we have to make sure the sql statement variables, resultset variables with in the procedures are not named the same or used multiple times. And also declare or initialize variables used in the stored procedure
lessons learnt by me and happy learning.
var var_Custkey=0;
while (res2.next())
{
var_Custkey =res2.getColumnValue(1)
return_value= return_value + var_Custkey;
if (var_CustKey == 0)
{
var_CustdimKey =1
}
else
{
var_CustdimKey=var_CustKey+1
}
return_value= return_value + var_CustdimKey.toString();
var SQL_INSDIMCUS="INSERT INTO
DIM_CUSTOMER(CUSTOMER_KEY,CUSTOMER_ID,CUSTOMER_NAME,CONSUMER_ID,CONSUMER_NAME,CUSTOMER_IND,TODATE)" + " SELECT DISTINCT :1, CUSTOMER_NUMBER,CUSTOMER_NAME,CONSUMER_NUMBER,CONSUMER_SHPPNG_FNAME + ' ' + CONSUMER_SHPPNG_LNAME AS CONSUMER_NAME, FALSE, NULL FROM STG_HOUSTONS WHERE CONSUMER_NUMBER = :2 ;";
var stmt = snowflake.createStatement({sqlText: SQL_INSDIMCUS,binds: [var_CustdimKey,var_ConsumNo]} );
var res3 = stmt.execute();
}
I am kinda stuck with the limitations of UDFs/stored Procedures in Snowflake and wonder what I am missing. I feel like there has to be a solution, but I can't see it right now, so maybe som new input from stackoverflow can help me.
I want something (UDF/Procedure) that has an input and returns a table. I created a stored procedure that generates the SQl-Statement. But in the stored procedure I can only return a single value and no table.
My stored procedure is the following:
CREATE OR REPLACE PROCEDURE DWH.TEMP.getAccounts(NODE_NAME VARCHAR)
RETURNS VARCHAR
LANGUAGE JAVASCRIPT
EXECUTE AS OWNER
AS '
var maxLevel = 0;
function executeSQL(SQL ) {
// Prepare statement.
var stmt = snowflake.createStatement(
{
sqlText: SQL
}
);
// Execute Statement
var res = stmt.execute();
res.next();
return res.getColumnValue(1);
}
maxLevel= executeSQL(" SELECT MAX(LEVEL) as Max_Level FROM DWH.MART.VDIM_ACCOUNT");
var sqlString = "SELECT DISTINCT AccountID FROM DWH.MART.VDIM_ACCOUNT WHERE ";
var sqlWhere = "";
for (var i = 0;i<=maxLevel;i++)
{
sqlWhere += "Group_L"+ i +" in (\'"+NODE_NAME+"\')";
if(i<maxLevel)
{
sqlWhere += " OR ";
}
}
sqlString += sqlWhere;
return sqlString;
'
If I execute this with a node, for example with call GETACCOUNTS('Test') I get the result:
SELECT DISTINCT AccountID
FROM dwh.mart.vdim_account
WHERE group_l0 IN ( 'TEST' )
OR group_l1 IN ( 'TEST' )
OR group_l2 IN ( 'TEST' )
OR group_l3 IN ( 'TEST' )
OR group_l4 IN ( 'TEST' )
OR group_l5 IN ( 'TEST' )
But I want the result of above SQL-Statement returnes as a table or list. How do I do this in Snowflake?
I am stuck because I need the for loop to dynamically generate my SQL-Statement. I can only use loops in JS UDFs, but I can only execute SQL in SQL UDFs.
i'm trying to get timestamp value as result of stored procedure.
but getting error .
error message:- SQL Error [100132] [P0000]: JavaScript execution error: Incorrect Timestamp returned.
CREATE OR REPLACE PROCEDURE simple_stored_procedure_example(awesome_argument VARCHAR)
returns TIMESTAMPNTZ
language javascript
as
$$
var cmd = "SELECT EXTRACT_END_TIME FROM EXPLARITY_DB.EXPLARITY_SCHEMA.DEMO_CONTROL_TABLE WHERE PIPELINE_NAME =:1";
var sql = snowflake.createStatement(
{sqlText: cmd,
binds: ['awesome_argument']
}
);
var result1 = sql.execute();
return result1;
$$;
CALL simple_stored_procedure_example('pipeline1');
It can be done like this:
create or replace table tst_tbl(c1 timestamp);
insert into tst_tbl values ('2021-02-02 10:00:00.000');
create or replace procedure my_test(myarg VARCHAR)
returns TIMESTAMP
language javascript
as
$$
var cmd = "SELECT c1 FROM tst_tbl";
var sql = snowflake.createStatement({sqlText: cmd});
var resultSet = sql.execute();
resultSet.next();
my_date = resultSet.getColumnValue(1);
return my_date;
$$;
call my_test('test');
I get 1 row back as expected.
I am getting a count(*) after joining two Snowflake tables. This is done inside a stored procedure. If the count is greater than zero, I need to pass a value. My stored procedure gets called from a NiFi Processor and I have to return the value to NiFi so that an email can be sent from NiFi.
I am getting 'NaN' as output for the below code.
CREATE OR REPLACE PROCEDURE test_Delete_excep()
returns float not null
language javascript
as
$$
var rs;
var return_value = 0;
var SQL_JOIN = "select count(*) from (Select GT.VARIANTDATA from GOV_TEST GT inner join GOV_TEST_H GTH on GT.VARIANTDATA:COL1::String = GTH.VARIANTDATA:COL1::String where to_char(GT.VARIANTDATA) != to_char(GTH.VARIANTDATA));";
var stmt = snowflake.createStatement({sqlText: SQL_JOIN});
rs = stmt.execute();
rs.next();
return_value += JSON.stringify(rs.getColumnValue(1));
if (return_value > 0) { return 'email required';}
$$;
Here is the result:
Row TEST_DELETE_EXCEP
1 NaN
How can I do the arithmetic calculation and return a value to NiFi processor?
You are never returning a float value, which the SP defines as the return type. If return_value is greater than 0, it will try to return the string 'email required.', which is not a float. That will generate a NaN. If return_value is not greater than 0, the code will never return a value of any kind. That will return NULL. Since you specify NOT NULL for the return, that will force it to NaN
Also, I'm not sure why you're trying to stringify the rs.getColumnValue(1). The select count(*) will produce an integer value, which you can read directly.
You probably want something like this:
CREATE OR REPLACE PROCEDURE test_Delete_excep()
returns float not null
language javascript
as
$$
var rs;
var return_value = 0;
var SQL_JOIN = "select count(*) from (Select GT.VARIANTDATA from GOV_TEST GT inner join GOV_TEST_H GTH on GT.VARIANTDATA:COL1::String = GTH.VARIANTDATA:COL1::String where to_char(GT.VARIANTDATA) != to_char(GTH.VARIANTDATA));";
var stmt = snowflake.createStatement({sqlText: SQL_JOIN});
rs = stmt.execute();
if(rs.next()) {
return_value = rs.getColumnValue(1);
} else {
return -1;
}
return return_value;
$$;
This will return the row count produced by your join SQL. If you want something different, please clarify the desired output.