青岛网站设计机构,有谁做过网站建设,网站开发如何设置背景图片,微博推广的好处背景
大数据平台的租户要使用udf#xff0c;他们用beeline连接#xff0c; 意味着要通过hs2#xff0c;但如果有多个hs2#xff0c;各个hs2之间不能共享#xff0c;需要先把文件传到hdfs#xff0c;然后手动在各hs2上create function。之后就可以永久使用了#xff0c;…背景
大数据平台的租户要使用udf他们用beeline连接 意味着要通过hs2但如果有多个hs2各个hs2之间不能共享需要先把文件传到hdfs然后手动在各hs2上create function。之后就可以永久使用了重启hs2也可以
调研
先查的hive官网
https://cwiki.apache.org/confluence/display/Hive/LanguageManualUDF#LanguageManualUDF-CreatingCustomUDFs 用beeline执行add jar 和create function但发现只在当前的hs2生效
然后查cdh官网
cdh的官网上说配UDF需要考虑是否重启hs2是否启用sentry列出了3种方案。 https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_mc_hive_udf.html Direct JAR reference configuration Straight-forward, but recommended for development only. Does not support Sentry. 试了下是永久的重启仍然生效但只对当前的hs2有效如果有多个hs2需要在每个hs2上都执行create function命令 虽然我们开了sentry但没影响sentry仍然有效
pom
project xmlnshttp://maven.apache.org/POM/4.0.0 xmlns:xsihttp://www.w3.org/2001/XMLSchema-instancexsi:schemaLocationhttp://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsdmodelVersion4.0.0/modelVersiongroupIdorg.example/groupIdartifactIdsm3UDF/artifactIdversion1.0/versionpackagingjar/packagingnamesm3UDF/nameurlhttp://maven.apache.org/urlpropertiesproject.build.sourceEncodingUTF-8/project.build.sourceEncoding/propertiesdependenciesdependencygroupIdorg.bouncycastle/groupIdartifactIdbcprov-jdk15on/artifactIdversion1.68/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-common/artifactIdversion3.1.1/version/dependency
!-- dependency--
!-- groupIdjunit/groupId--
!-- artifactIdjunit/artifactId--
!-- version4.13.2/version--
!-- scopetest/scope--
!-- /dependency--dependencygroupIdorg.apache.hive/groupIdartifactIdhive-exec/artifactIdversion2.1.1-cdh6.3.2/version/dependency/dependenciesbuildpluginsplugingroupIdorg.apache.maven.plugins/groupIdartifactIdmaven-compiler-plugin/artifactIdversion3.1/versionconfigurationsource1.8/sourcetarget1.8/target/configuration/plugin/plugins/build
/project
java
package org.picc.encrypt;import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.io.Text;
import org.bouncycastle.crypto.digests.SM3Digest;
import org.apache.hadoop.hive.ql.exec.UDF;public class Sm3Fun extends UDF{public static String sm3(String saltBefore, String text, String saltAfter) {if (text null) {return null;}Text result new Text();SM3Digest digest new SM3Digest();Text sb new Text(saltBefore);Text value new Text(text);Text sa new Text(saltAfter);byte[] hashData new byte[32];digest.reset();digest.update(sb.getBytes(), 0, sb.getLength());digest.update(value.getBytes(), 0, value.getLength());digest.update(sa.getBytes(), 0, sa.getLength());digest.doFinal(hashData, 0);String sm3Hex Hex.encodeHexString(hashData);result.set(sm3Hex);return result.toString();}public String evaluate(String text) {if (text null) {return null;}Text result new Text();SM3Digest digest new SM3Digest();Text value new Text(text);byte[] hashData new byte[32];digest.reset();digest.update(value.getBytes(), 0, value.getLength());digest.doFinal(hashData, 0);String sm3Hex Hex.encodeHexString(hashData);result.set(sm3Hex);return result.toString();}
}
