Visualize the data lineage after analyzing the input SQL query.
input: SQL text
output: data lineage diagram
Find the example code under the directory:
└── 1\
$(async () => {// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:315, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');// set sql text propertysqlflow.sqltext.set(`CREATE VIEW vsal AS SELECT a.deptno "Department", a.num_emp / b.total_count "Employees", a.sal_sum / b.total_sal "Salary" FROM (SELECT deptno, Count() num_emp, SUM(sal) sal_sum FROM scott.emp WHERE city = 'NYC' GROUP BY deptno) a, (SELECT Count() total_count, SUM(sal) total_sal FROM scott.emp WHERE city = 'NYC') b ;`);sqlflow.visualize();});
2. Visualize job
Visualize the data lineage in a SQLFlow Job. The SQLFlow job must be already created.
input: a SQLFlow job id, or leave it empty to view the latest job
output: data lineage diagram
Find the example code under the directory:
└── 2\
$(async () => {constsqlflow=awaitSQLFlow.init({ container:document.getElementById('demo-2'), width:1000, height:800, apiPrefix:'http://xxx.com/api', });// view job detail by job id, or leave it empty to view the latest jobawaitsqlflow.job.lineage.viewDetailById('b38273ec356d457bb98c6b3159c53be3');});
3. Visualize job with hard coded parameters
Visualize the data lineage of a specified table or column in a SQLFlow job.
input: a SQLFlow job id, or leave it empty to view the latest job
input: database, schema, table, column.
If the column is omitted, the data lineage for the specified table will be returned.
if the table and column are ommited, the data lineage for the specified schema will be returned.
if the schema, table and column are ommited, the data lineage for the specified database will be returned.
output: data lineage diagram
Find the example code under the directory:
└── 3\
$(async () => {constsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:700, apiPrefix:'http://xxx.com/api', });// view job detail by job id, or leave it empty to view the latest jobawaitsqlflow.job.lineage.viewDetailById('b38273ec356d457bb98c6b3159c53be3');sqlflow.job.lineage.selectGraph({ database:'DWDB',// schema:'DBO', table:'#TMP', column:'NUMBER_OFOBJECTS', });});
4. Visualize job with parameters
Visualize the data lineage of a specified table or column in a SQLFlow job.
input: a SQLFlow job id, or leave it empty to view the latest job
input: database, schema, table, column.
If the column is omitted, the data lineage for the specified table will be returned.
if the table and column are ommited, the data lineage for the specified schema will be returned.
if the schema, table and column are ommited, the data lineage for the specified database will be returned.
output: data lineage diagram
Find the example code under the directory:
└── 4\
$(async () => {const$jobid=$('#jobid');const$database=$('#database');const$schema=$('#schema');const$table=$('#table');const$column=$('#column');const$recordset=$('#recordset');const$function=$('#function');const$visualize=$('#visualize');constsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1200, height:800, apiPrefix:'http://xxx.com/api', token:'',// input your token });constvisualize=async () => {// reset default option$recordset.prop('checked',false);$function.prop('checked',false);// view job detail by job id, or leave it empty to view the latest jobawaitsqlflow.job.lineage.viewDetailById($jobid.val());sqlflow.job.lineage.selectGraph({ database:$database.val(), schema:$schema.val(), table:$table.val(), column:$column.val(), }); };visualize();$visualize.click(visualize);$recordset.change(() => {constchecked=$recordset.prop('checked');sqlflow.setting.showIntermediateRecordset.set(checked); });$function.change(() => {constchecked=$function.prop('checked');sqlflow.setting.showFunction.set(checked); });});
5. Set data lineage options of SQL query
Generate the data lineage of a SQL query with different input parameters.
Find the example code under the directory:
└── 5\
$(async () => {const$sqltext=$('#sqltext');const$dataflow=$('#dataflow');const$impact=$('#impact');const$args=$('#args');const$recordset=$('#recordset');const$function=$('#function');const$constant=$('#constant');const$transform=$('#transform');const$visualize=$('#visualize');const$visualizeTableLevel=$('#visualizeTableLevel');// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:800, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');constvisualize=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());// set default options$recordset.prop('checked',true);$dataflow.prop('checked',true);$args.prop('checked',true);$impact.prop('checked',false);$function.prop('checked',false);$constant.prop('checked',true);$transform.prop('checked',false);sqlflow.visualize(); };visualize();$visualize.click(visualize);constvisualizeTableLevel=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());sqlflow.visualizeTableLevel(); };$visualizeTableLevel.click(visualizeTableLevel);$dataflow.change(() => {constchecked=$dataflow.prop('checked');sqlflow.setting.dataflow.set(checked); });$impact.change(() => {constchecked=$impact.prop('checked');sqlflow.setting.impact.set(checked); });$args.change(() => {constchecked=$args.prop('checked');sqlflow.setting.dataflowOfAggregateFunction.set(checked); });$recordset.change(() => {constchecked=$recordset.prop('checked');sqlflow.setting.showIntermediateRecordset.set(checked); });$function.change(() => {constchecked=$function.prop('checked');sqlflow.setting.showFunction.set(checked); });$constant.change(() => {constchecked=$constant.prop('checked');sqlflow.setting.showConstant.set(checked); });$transform.change(() => {constchecked=$transform.prop('checked');sqlflow.setting.showTransform.set(checked); });});
6. Visualize an embedded json object in html page
SQLFlow will visualize the json object which contains the data lineage information and will show the actionable diagram.
Since all layout data is included in the json file, the SQLFlow widget will draw the diagram and needn't to connect to the SQLFlow backend.
So this SQLFlow widget can work without the installation of the Gudu SQLFlow.
Find the example code under the directory:
└── 6\
7. Visualize data lineage in a separate json file
Read and visualize the data lineage in a json file. This json file should be accessable in the same server as the SQLFlow widget.
Since all layout data is included in the json file, the SQLFlow widget will generate the diagram and needn't to connect to the SQLFlow backend.
This SQLFlow widget can work without the installation of the Gudu SQLFlow.
Find the example code under the directory:
└── 7\
8. Get error message
Getting error message after processing input SQL qurey.
Find the example code under the directory:
└── 8\
$(async () => {const$sqltext=$('#sqltext');const$visualize=$('#visualize');const$error=$('#error');// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:400, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');constvisualize=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());awaitsqlflow.visualize();consterror=sqlflow.error.get();if (error &&error.length>0) {console.log(sqlflow.error.get());let text ='';error.forEach(item => { text +=`${item.errorType} : ${item.errorMessage}`; });$error.val(text); } else {$error.val(''); } };visualize();$visualize.click(visualize);});
9. Event: add an event listener on field(column) click
Add an event listener on field(column) click so that you can get detailed information about the field(column) that been clicked.
Find the example code under the directory:
└── 9\
$(async () => {const$sqltext=$('#sqltext');const$visualize=$('#visualize');const$message=$('#message');// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:400, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');// add an event listener on field(column) clicksqlflow.addEventListener('onFieldClick', field => {$message.val(JSON.stringify(field));// remove all event listeners// sqlflow.removeAllEventListener() });constvisualize=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());awaitsqlflow.visualize(); };visualize();$visualize.click(visualize);});
12. Access data lineage from url
User can directly access the data lineage through an url by specifying the data lineage type, table and column.
All data lineages come from the default job at the Gudu SQLFlow backend. If no default job is set, the lineage data will be retrieved from the latest job.
column: if column is omitted, return the lineage for table
stopat: value in regex expression, the lineage(upstream or downstream) stops at the given table if this input is set
the table and column name in the url is case insensitive.
stopat with regex expression
? The question mark indicates zero or one occurrences of the preceding element. For example,colou?r matches both "color" and "colour". The question mark equals to {0, 1}
* The asterisk indicates zero or more occurrences of the preceding element. For example,ab*c matches "ac", "abc", "abbc", "abbbc", and so on. The asterisk equals to {0,}
+ The plus sign indicates one or more occurrences of the preceding element. For example,ab+c matches "abc", "abbc", "abbbc", and so on, but not "ac".
Regular expression has two matching modes: matches and find.
matches are all exact matches, find is a partial match. SQLFlow uses matches mode.
Example:
To match mio.public.usecase_na_mio004_infohubview006 with base string mio.public.usecase_na_mio004_infohub
The regex expression should be: mio.public.usecase_na_mio004_infohub.*
. matches any single character, * matches the preceding element zero or more times(in this case, it can be any character).
we cannot use mio.public.usecase_na_mio004_infohub.? to match because ? indicates only zero or one occurrences of the preceding element, it is not a exact match:
$(async () => {const$sqltext=$('#sqltext');const$dataflow=$('#dataflow');const$impact=$('#impact');const$recordset=$('#recordset');const$function=$('#function');const$visualize=$('#visualize');const$file=$('#file');// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:800, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');constvisualize=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());// set default options$recordset.prop('checked',true);$dataflow.prop('checked',true);$impact.prop('checked',false);$function.prop('checked',false);sqlflow.visualize(); };$file.on('change',async e => {constfile=$file.prop('files')[0];constcontent=awaitfile.text();$sqltext.val(content);visualize(); });$visualize.click(visualize);$dataflow.change(() => {constchecked=$dataflow.prop('checked');sqlflow.setting.dataflow.set(checked); });$impact.change(() => {constchecked=$impact.prop('checked');sqlflow.setting.impact.set(checked); });$recordset.change(() => {constchecked=$recordset.prop('checked');sqlflow.setting.showIntermediateRecordset.set(checked); });$function.change(() => {constchecked=$function.prop('checked');sqlflow.setting.showFunction.set(checked); });});
14. Visualize the lineage data using Vue
SQLFlow Widget provides a Vue library to support Vue framework.
Find the example code under the directory:
└── 14\
document.addEventListener('DOMContentLoaded',async () => {Vue.component('sqlflow', { template:'<div ref="el"></div>',asyncmounted() {// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:this.$refs.el,// get element ref from vue width:1000, height:315, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');// set sql text propertysqlflow.sqltext.set(`CREATE VIEW vsal AS SELECT a.deptno "Department", a.num_emp / b.total_count "Employees", a.sal_sum / b.total_sal "Salary" FROM (SELECT deptno, Count() num_emp, SUM(sal) sal_sum FROM scott.emp WHERE city = 'NYC' GROUP BY deptno) a, (SELECT Count() total_count, SUM(sal) total_sal FROM scott.emp WHERE city = 'NYC') b ;`);sqlflow.visualize(); }, });var app =newVue({ el:'#sqlflow', template:'<sqlflow />', });});
15. Event: add an event listener on table click
Add an event listener on table click so that you can get detailed information about the table which is clicked.
Find the example code under the directory:
└── 15\
$(async () => {const$sqltext=$('#sqltext');const$visualize=$('#visualize');const$message=$('#message');// get a instance of SQLFlowconstsqlflow=awaitSQLFlow.init({ container:document.getElementById('sqlflow'), width:1000, height:400, apiPrefix:'http://xxx.com/api', token:'',// input your token });// set dbvendor propertysqlflow.vendor.set('oracle');// add an event listener on table clicksqlflow.addEventListener('onTableNameClick', table => {$message.val(JSON.stringify(table));// remove all event listeners// sqlflow.removeAllEventListener() });constvisualize=async () => {// set sql text propertysqlflow.sqltext.set($sqltext.val());awaitsqlflow.visualize(); };visualize();$visualize.click(visualize);});