Snowflake Stored Procedure Dynamic Column Pivot - stored-procedures

I am trying to create a stored procedure in Snowflake that pivot's the data in a derived table. The pivot columns are dynamic in nature. I have found a way to do this through parameter passing. I am trying to do the same without passing any parameters but the code is not working.
My approach has been to apply a loop at the table from which I am extracting the columns into a variable. Then pass this variable in the pivot. The while loop by itself seems to be working fine. The error comes when this variable is being passed to the pivot.
The code I am using:
CREATE OR REPLACE PROCEDURE LOOP_EXMPL_5()
returns varchar
language javascript
as
$$
var column1 = 'qwerty';
var command = `SELECT ATTR_NAME FROM TBL_DIM`;
var stmt = snowflake.createStatement({sqlText: command});
var rs = stmt.execute();
while (rs.next())
{
var column1 = column1.concat(",","'",rs.getColumnValue(1),"'");
}
var column2 = column1
var command_1 = `CREATE OR REPLACE VIEW HIERARCHY_VIEW_2 AS SELECT * FROM (SELECT MSTR.PROD_CODE AS
PROD_CODE,DIM.ATTR_NAME AS ATTR_NAME,MSTR.ATTR_VALUE AS ATTR_VALUE FROM TBL_DIM DIM INNER JOIN
TBL_MSTR MSTR ON DIM.ATTR_KEY=MSTR.ATTR_KEY ) Q
PIVOT (MAX (Q.ATTR_VALUE) FOR Q.ATTR_NAME IN ('${column2}'))
AS P
ORDER BY P.PROD_CODE;`;
var stmt_1 = snowflake.createStatement({sqlText: command_1});
var rs_1 = stmt_1.execute();
return 'success'
$$;
The error I am getting:
Execution error in store procedure LOOP_EXMPL_5: SQL compilation error: syntax error line 2 at position 73 unexpected 'Region'. At Statement.execute, line 16 position 21.
The variable value that is being passed:
qwerty,'Region','Sub-Region','Division','Company-Product','Company-Mfg','Company-Ship From to Customer','Business Unit','Category','Sub-Category','Segment','Sub-Segment','Brand','Aggregate Brand','Sub-Brand','PPG'
I will remove the qwerty part in the SQL somehow.

Here is the working code. Thanks a lot for your help Felipe and Greg.
CREATE OR REPLACE PROCEDURE LOOP_EXMPL_9()
returns varchar
language javascript
as
$$
var column1 = "";
var command = `SELECT ATTR_NAME FROM TBL_DIM`;
var stmt = snowflake.createStatement({sqlText: command});
var rs = stmt.execute();
while (rs.next())
{
if (column1 != "") column1 += ",";
column1 += `'${rs.getColumnValue (1)}'`;
}
var column2 = column1;
var command_1 = `CREATE OR REPLACE VIEW HIERARCHY_VIEW_2 AS SELECT * FROM (SELECT
MSTR.PROD_CODE AS PROD_CODE,DIM.ATTR_NAME AS ATTR_NAME,MSTR.ATTR_VALUE AS ATTR_VALUE
FROM TBL_DIM DIM INNER JOIN TBL_MSTR MSTR ON DIM.ATTR_KEY=MSTR.ATTR_KEY ) Q
PIVOT (MAX (Q.ATTR_VALUE) FOR Q.ATTR_NAME IN (${column2}))
AS P
ORDER BY P.PROD_CODE;`;
var stmt_1 = snowflake.createStatement({sqlText: command_1});
var rs_1 = stmt_1.execute();
return 'success'
$$;

For your objectives, see:
https://medium.com/snowflake/dynamic-pivots-in-sql-with-snowflake-c763933987c
How to pivot on dynamic values in Snowflake
To debug the current problem: column1.concat(",","'",rs.getColumnValue(1),"'"); is not escaping and quoting the column names correctly.
Set the UDF to return that value and then the value of var command_1 so you can debug appropriately.
A good way to escape the columns:
select '\\''
|| listagg(distinct pivot_column, '\\',\\'') within group (order by pivot_column)
|| '\\''
And then use:
for pivot_column in (${col_list}))

You can start with an empty string:
var column1 = "";
You can then concatenate the column list like this:
if (column1 != "") column1 += ",";
column1 += `"${rs.getColumnValue(1)}"`);
The reason you're getting the SQL syntax error is that the column names are in single quotes when they should be in double quotes.

Related

Create master view from array of tables passed as input to stored procedure Snowflake

I want to create a master view combining all the tables passed as input to Snowflake Stored procedure. Please help on how the code can be framed for this.
create or replace procedure TEST_PROC("SRC_DB" VARCHAR(30),
"SRC_SCHEMA" VARCHAR(30), "TGT_DB" VARCHAR(30), "TGT_SCHEMA"
VARCHAR(30))
RETURNS varchar
LANGUAGE JAVASCRIPT
EXECUTE AS OWNER
as
$$
var result = '';
var tab = 'TABLE1,TABLE2'
var get_tables = `
with cte as(select value from table(SPLIT_TO_TABLE
(('${tab}'),','))
) select value from cte;`
var tables_name_master=snowflake.execute ({sqlText: get_tables});
var lcols_agg = '';
while(tables_name_master.next()){
var table_value = tables_name_master.getColumnValue(1);
var column_list = `
WITH cte2 as (select COLUMN_NAME , listagg(TABLE_NAME, ', ')
within group (order by COLUMN_NAME) A
from ${SRC_DB}.information_schema.COLUMNS
where TABLE_SCHEMA= '${SRC_SCHEMA}' and TABLE_NAME in (select
value from table(SPLIT_TO_TABLE (('${tab}'),',')))
group by COLUMN_NAME order by COLUMN_NAME
),
cte3 as (select 'x' x, COLUMN_NAME,iff(contains(A,'${table_value}'),COLUMN_NAME,CONCAT('NULL AS \"',COLUMN_NAME,'\"')) valuess from cte2 order by COLUMN_NAME
)select listagg(valuess,',') final FROM cte3 GROUP BY x
`;
var rs = snowflake.execute({ sqlText: column_list });
while(rs.next()){
lcols_agg += "SELECT " + rs.getColumnValue(1) + " FROM "+ SRC_DB+"."+SRC_SCHEMA+"."+tables_name_master.getColumnValue(1) + "\n" +"UNION " +"\n"
}
}
var count1 = 0 ;
count1 = lcols_agg.length
result = lcols_agg.substring(0,(count1-7));
const create_union_view = `
create or replace view abcd AS ${result}
;`
var view_create = snowflake.execute({ sqlText: create_union_view });
view_create.next()
return result
$$
;
call SP_TEST('ABC','DEF','PQR','STU');
THis generates my final view statement as
CREATE OR REPLACE VIEW ABCD AS
COL1,COL2,COL3
UNION
NULL AS "COL2",NULL AS "COL3",COL1
Now due to mismatch of order od columns in union the view is throwing error while we do select * from abcd, any way we can have the columns of both tables in same order or any other work around?
You need to ORDER BY the function listagg after the cte3 expression:
instead of
select listagg(valuess,',') final FROM cte3 GROUP BY x
change it to:
select listagg(valuess,',') within group (order by valuess) final FROM cte3 GROUP BY x
See if that helps to resolve the order issue.

Execution error in stored procedure : Unsupported type for binding argument 2undefined

I am executing
CALL LOAD_CUSTDIM_HOUSTONS:
This does the below:
I am running this code to get unique Customer Details such as Consumer No and Customer Name from STG_HOUSTON table and Get Max(CUST_DImKEY) from the DIM_CUSTOMER Key for the CUSTOMER Name from above i.e Houston's.
Next I am checking if the Max key value is null i mean no match for the CUSTOMER so Key value first time to be always 1 next time it will be max value + 1.
Once we get this bind this value and the select where clause will get the unique customer and insert into Customer Dimension table.
Stored procedure looks like this:
create or replace procedure LOAD_CUSTDIM_HOUSTONS()
returns varchar not null
language javascript
as
$$
var return_value = "";
try
{
var SQL_SMT = "SELECT DISTINCT CONSUMER_NUMBER,CUSTOMER_NAME FROM STG_HOUSTONS ORDER BY CONSUMER_NUMBER ASC";
var stmt1 = snowflake.createStatement(
{
sqlText: SQL_SMT
}
);
/* Creates result set */
var res1 = stmt1.execute();
while (res1.next())
{
var var_ConsumNo=res1.getColumnValue(1);
var var_custname=res1.getColumnValue(2);
return_value =" Inside While";
var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
var res2 = t_stmt.execute();
res2.next();
var_Custkey =res2.getColumnValue(1)
return_value= return_value + var_Custkey;
if (var_CustKey == null)
{
var_CustdimKey =1
}
else
{
var_CustdimKey=var_CustKey+1
}
return_value= return_value + var_CustdimKey.toString();
var SQL_INSDIMCUS="INSERT INTO DIM_CUSTOMER(CUSTOMER_KEY,CUSTOMER_ID,CUSTOMER_NAME,CONSUMER_ID,CONSUMER_NAME,CUSTOMER_IND,TODATE)" + " SELECT DISTINCT :1, CUSTOMER_NUMBER,CUSTOMER_NAME,CONSUMER_NUMBER,CONSUMER_SHPPNG_FNAME + ' ' + CONSUMER_SHPPNG_LNAME AS CONSUMER_NAME, FALSE, NULL FROM STG_HOUSTONS WHERE CONSUMER_NUMBER = :2 ;";
var stmt = snowflake.createStatement({sqlText: SQL_INSDIMCUS,binds: [var_CustdimKey,var_ConsumNo]} );
var res3 = stmt.execute();
result = "COMPLETED SUCCESSFULLY!";
}
}
catch (err)
{ result = "Failed!";
var time_st = snowflake.execute( {sqlText: `SELECT CURRENT_TIMESTAMP;`} );
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
});
}
return return_value;
$$
;
This is the function I am using it to get max(DIM CUST KEY).
create or replace function GET_CUSTDIMKEYS ( CUSTOMERNAME varchar )
returns bigint
as 'Select max(CUSTOMER_KEY)
from PUBLIC.DIM_CUSTOMER
where CUSTOMER_NAME = CUSTOMERNAME ';
I tried with Binding variable / direct substitution of the values to dynamically create the SQL and execute it.
Using DBeaver21.1.1 IDE to create snowflake stored procedure and functions
This is the execution log Created by the IDE.
2021-10-05 20:44:48.030 - SQL Error [100183] [P0000]: Execution error in store procedure LOAD_CUSTDIM_HOUSTONS:
Unsupported type for binding argument 2undefined
At Snowflake.execute, line 49 position 14
org.jkiss.dbeaver.model.sql.DBSQLException: SQL Error [100183] [P0000]: Execution error in store procedure LOAD_CUSTDIM_HOUSTONS:
Unsupported type for binding argument 2undefined
At Snowflake.execute, line 49 position 14
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCPreparedStatementImpl.executeStatement(JDBCPreparedStatementImpl.java:208)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeStatement(SQLQueryJob.java:510)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.lambda$0(SQLQueryJob.java:441)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:171)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeSingleQuery(SQLQueryJob.java:428)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.extractData(SQLQueryJob.java:813)
at org.jkiss.dbeaver.ui.editors.sql.SQLEditor$QueryResultsContainer.readData(SQLEditor.java:3280)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.lambda$0(ResultSetJobDataRead.java:118)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:171)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.run(ResultSetJobDataRead.java:116)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetViewer$ResultSetDataPumpJob.run(ResultSetViewer.java:4624)
at org.jkiss.dbeaver.model.runtime.AbstractJob.run(AbstractJob.java:105)
at org.eclipse.core.internal.jobs.Worker.run(Worker.java:63)
Line 49 is:
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
The error message:
Unsupported type for binding argument 2undefined
means the variable passed into the binding argument is not defined.
This tells me that the variable "err" was not initialized properly. I have seen some cases that under certain conditions, like when calling a function that does not exist on an object, though exception is thrown, the argument passed into the catch block is still NULL.
You need to look into the query history for this particular session, and find out which one of the queries that ran from the SP failed, and find out why.
And in the catch block, try to check the "err" object before using its attributes, something like:
if (err) {
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS',err.code, err.state,err.message, err.stackTraceTxt]
});
} else {
snowflake.execute({
sqlText: `insert into PROC_ERROR_LOG
VALUES (?,?,?,?,?)`
,binds: ['LOAD CUSTOMER DIMENSION HOUSTONS','my code', 'my state','my error', 'my stack']
});
}
This can avoid such failure.
This is in addition to Eric's answer. You should not wrap strings in single quotes when used as bind variables:
var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
This should be:
//var_custname = "'" + var_custname + "'";
var t_SQLstmt = " SELECT GET_CUSTDIMKEYS(?)" ;
var t_stmt = snowflake.createStatement ({sqlText: t_SQLstmt,binds: [var_custname]});
If you wrap a string type bind variable in single quotes, the single quotes become part of the variable. The practical effect is that instead of running a statement like this:
SELECT GET_CUSTDIMKEYS('MyCustomer'); -- Looks for MyCustomer
You run a statement like this:
SELECT GET_CUSTDIMKEYS('\'MyCustomer\''); -- Looks for 'MyCustomer'
The issue is first the second result set should have a while loop which was missing,
And rest of the code proceeds will solve the issue and also we have to make sure the sql statement variables, resultset variables with in the procedures are not named the same or used multiple times. And also declare or initialize variables used in the stored procedure
lessons learnt by me and happy learning.
var var_Custkey=0;
while (res2.next())
{
var_Custkey =res2.getColumnValue(1)
return_value= return_value + var_Custkey;
if (var_CustKey == 0)
{
var_CustdimKey =1
}
else
{
var_CustdimKey=var_CustKey+1
}
return_value= return_value + var_CustdimKey.toString();
var SQL_INSDIMCUS="INSERT INTO
DIM_CUSTOMER(CUSTOMER_KEY,CUSTOMER_ID,CUSTOMER_NAME,CONSUMER_ID,CONSUMER_NAME,CUSTOMER_IND,TODATE)" + " SELECT DISTINCT :1, CUSTOMER_NUMBER,CUSTOMER_NAME,CONSUMER_NUMBER,CONSUMER_SHPPNG_FNAME + ' ' + CONSUMER_SHPPNG_LNAME AS CONSUMER_NAME, FALSE, NULL FROM STG_HOUSTONS WHERE CONSUMER_NUMBER = :2 ;";
var stmt = snowflake.createStatement({sqlText: SQL_INSDIMCUS,binds: [var_CustdimKey,var_ConsumNo]} );
var res3 = stmt.execute();
}

UDF/Procedure with control flow that returns a table

I am kinda stuck with the limitations of UDFs/stored Procedures in Snowflake and wonder what I am missing. I feel like there has to be a solution, but I can't see it right now, so maybe som new input from stackoverflow can help me.
I want something (UDF/Procedure) that has an input and returns a table. I created a stored procedure that generates the SQl-Statement. But in the stored procedure I can only return a single value and no table.
My stored procedure is the following:
CREATE OR REPLACE PROCEDURE DWH.TEMP.getAccounts(NODE_NAME VARCHAR)
RETURNS VARCHAR
LANGUAGE JAVASCRIPT
EXECUTE AS OWNER
AS '
var maxLevel = 0;
function executeSQL(SQL ) {
// Prepare statement.
var stmt = snowflake.createStatement(
{
sqlText: SQL
}
);
// Execute Statement
var res = stmt.execute();
res.next();
return res.getColumnValue(1);
}
maxLevel= executeSQL(" SELECT MAX(LEVEL) as Max_Level FROM DWH.MART.VDIM_ACCOUNT");
var sqlString = "SELECT DISTINCT AccountID FROM DWH.MART.VDIM_ACCOUNT WHERE ";
var sqlWhere = "";
for (var i = 0;i<=maxLevel;i++)
{
sqlWhere += "Group_L"+ i +" in (\'"+NODE_NAME+"\')";
if(i<maxLevel)
{
sqlWhere += " OR ";
}
}
sqlString += sqlWhere;
return sqlString;
'
If I execute this with a node, for example with call GETACCOUNTS('Test') I get the result:
SELECT DISTINCT AccountID
FROM dwh.mart.vdim_account
WHERE group_l0 IN ( 'TEST' )
OR group_l1 IN ( 'TEST' )
OR group_l2 IN ( 'TEST' )
OR group_l3 IN ( 'TEST' )
OR group_l4 IN ( 'TEST' )
OR group_l5 IN ( 'TEST' )
But I want the result of above SQL-Statement returnes as a table or list. How do I do this in Snowflake?
I am stuck because I need the for loop to dynamically generate my SQL-Statement. I can only use loops in JS UDFs, but I can only execute SQL in SQL UDFs.

return timestamp value in store procedure

i'm trying to get timestamp value as result of stored procedure.
but getting error .
error message:- SQL Error [100132] [P0000]: JavaScript execution error: Incorrect Timestamp returned.
CREATE OR REPLACE PROCEDURE simple_stored_procedure_example(awesome_argument VARCHAR)
returns TIMESTAMPNTZ
language javascript
as
$$
var cmd = "SELECT EXTRACT_END_TIME FROM EXPLARITY_DB.EXPLARITY_SCHEMA.DEMO_CONTROL_TABLE WHERE PIPELINE_NAME =:1";
var sql = snowflake.createStatement(
{sqlText: cmd,
binds: ['awesome_argument']
}
);
var result1 = sql.execute();
return result1;
$$;
CALL simple_stored_procedure_example('pipeline1');
It can be done like this:
create or replace table tst_tbl(c1 timestamp);
insert into tst_tbl values ('2021-02-02 10:00:00.000');
create or replace procedure my_test(myarg VARCHAR)
returns TIMESTAMP
language javascript
as
$$
var cmd = "SELECT c1 FROM tst_tbl";
var sql = snowflake.createStatement({sqlText: cmd});
var resultSet = sql.execute();
resultSet.next();
my_date = resultSet.getColumnValue(1);
return my_date;
$$;
call my_test('test');
I get 1 row back as expected.

How to do mathematical computation in stored procedure in snowflake?

I am getting a count(*) after joining two Snowflake tables. This is done inside a stored procedure. If the count is greater than zero, I need to pass a value. My stored procedure gets called from a NiFi Processor and I have to return the value to NiFi so that an email can be sent from NiFi.
I am getting 'NaN' as output for the below code.
CREATE OR REPLACE PROCEDURE test_Delete_excep()
returns float not null
language javascript
as
$$
var rs;
var return_value = 0;
var SQL_JOIN = "select count(*) from (Select GT.VARIANTDATA from GOV_TEST GT inner join GOV_TEST_H GTH on GT.VARIANTDATA:COL1::String = GTH.VARIANTDATA:COL1::String where to_char(GT.VARIANTDATA) != to_char(GTH.VARIANTDATA));";
var stmt = snowflake.createStatement({sqlText: SQL_JOIN});
rs = stmt.execute();
rs.next();
return_value += JSON.stringify(rs.getColumnValue(1));
if (return_value > 0) { return 'email required';}
$$;
Here is the result:
Row TEST_DELETE_EXCEP
1 NaN
How can I do the arithmetic calculation and return a value to NiFi processor?
You are never returning a float value, which the SP defines as the return type. If return_value is greater than 0, it will try to return the string 'email required.', which is not a float. That will generate a NaN. If return_value is not greater than 0, the code will never return a value of any kind. That will return NULL. Since you specify NOT NULL for the return, that will force it to NaN
Also, I'm not sure why you're trying to stringify the rs.getColumnValue(1). The select count(*) will produce an integer value, which you can read directly.
You probably want something like this:
CREATE OR REPLACE PROCEDURE test_Delete_excep()
returns float not null
language javascript
as
$$
var rs;
var return_value = 0;
var SQL_JOIN = "select count(*) from (Select GT.VARIANTDATA from GOV_TEST GT inner join GOV_TEST_H GTH on GT.VARIANTDATA:COL1::String = GTH.VARIANTDATA:COL1::String where to_char(GT.VARIANTDATA) != to_char(GTH.VARIANTDATA));";
var stmt = snowflake.createStatement({sqlText: SQL_JOIN});
rs = stmt.execute();
if(rs.next()) {
return_value = rs.getColumnValue(1);
} else {
return -1;
}
return return_value;
$$;
This will return the row count produced by your join SQL. If you want something different, please clarify the desired output.

Resources