手机编程的app,烟台seo做的好的网站,中国建设银行个人网站登录,图片放大网站关于Java连接Hive#xff0c;Spark等服务的Kerberos工具类封装
idea连接服务器的hive等相关服务的kerberos认证注意事项
idea 本地配置#xff0c;连接服务器#xff1b;进行kerberos认证#xff0c;连接hive、HDFS、Spark等服务注意事项#xff1a;
本地idea连接Hadoo…关于Java连接HiveSpark等服务的Kerberos工具类封装
idea连接服务器的hive等相关服务的kerberos认证注意事项
idea 本地配置连接服务器进行kerberos认证连接hive、HDFS、Spark等服务注意事项
本地idea连接Hadoop需要在本地安装Hadoop的window工具hadoop-3.1.1-winutils-master 配置环境变量配置hosts主机名映射kerberos认证需要在idea工作目录所在的磁盘的根目录下创建对应的文件夹把keytab放到该目录下,方便认证。krb5.conf放到对应的目录如system.properties中配置了krbConf/etc/krb5.conf;在项目所在的磁盘根目录下创建对应的etc目录在下面放配置文件krb5.conf。如我的idea工作空间在D盘那么就在D盘根目录下创建。在resource目录下放置集群的配置文件hdfs-site.xml、core-site.xml、mapred-site.xml、yarn-site.xml、hive-site.xml配置文件。认证注意事项如果最终是hive用户认证的那么生成的文件默认为hive的家目录如果是hdfs用户认证的生成的文件默认为hdfs的家目录。
properties工具类
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.Properties;/*** properties工具类*/public class PropertiesUtil {private static Log log LogFactory.getLog(PropertiesUtil.class);private static Properties propsnew Properties();private static String propertyFileName /system.properties;static {try {if (props.size() 0) {log.info(Start read the constv.properties file);InputStream input PropertiesUtil.class.getResourceAsStream(propertyFileName);props.load(input);input.close();}}catch (IOException ioe) {log.error(ioe.getMessage());log.debug(ioe);}}public static Integer getRequiredIntegerProperty(String propertyName){String str getRequiredStringProperty(propertyName);return Integer.parseInt(str);}public static String getRequiredStringProperty(String propertyName){String str getStringProperty(propertyName, null);if (StringUtils.isBlank(str)){throw new RuntimeException(propertyNamenot is property file propertyFileName);}return str;}public static String getStringProperty(String propertyName,String defaultValue){if (props.containsKey(propertyName) true){return (String) props.get(propertyName);}return defaultValue;}public static String getIntegerProperty(String propertyName,String defaultValue, String encoding){if (props.containsKey(propertyName) true){//编码转换从ISO8859-1转向指定的编码String value (String) props.get(propertyName);try{value new String(value.getBytes(ISO8859-1), encoding);}catch (UnsupportedEncodingException e){e.printStackTrace();}return value;}return defaultValue;}}线程池调度工具类
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;public class ScheduledThreadFactory implements ThreadFactory {private static final AtomicInteger poolNumber new AtomicInteger(1);private final ThreadGroup group;private final AtomicInteger threadNumber new AtomicInteger(1);private final String namePrefix;public ScheduledThreadFactory() {SecurityManager sSystem.getSecurityManager();group (s ! null) ? s.getThreadGroup(): Thread.currentThread().getThreadGroup();namePrefix Scheduled Pool- poolNumber.getAndIncrement()-Thread-;}public Thread newThread(Runnable r) {Thread t new Thread(group,r,namePrefixthreadNumber.getAndIncrement());/** 设置为守护进程所在的jar执行完就退出如果不是守护进程在linux运行时即使业务进程执行完成这个认证进程也不会关闭。* */t.setDaemon(true);//这个是线程默认的优先级 Thread.NORM_PRIORITYif (t.getPriority() ! Thread.NORM_PRIORITY){t.setPriority(Thread.NORM_PRIORITY);}return t;}}Kerberos认证工具类
import com.xxxx.utils.PropertiesUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;public class KerberosAuthen {private static ScheduledExecutorService scheduledExecutor Executors.newScheduledThreadPool(1,new ScheduledThreadFactory());public static void kerberosAuthen(){krbAuth();/** 每5分钟执行一次向kerberos进行认证的方法* */scheduledExecutor.scheduleAtFixedRate(()-krbAuth(),5L,5L, TimeUnit.MINUTES);}/*向kerberos认证* */private static void krbAuth(){String krbConf PropertiesUtil.getRequiredStringProperty(krb.conf);String krbKeytab PropertiesUtil.getRequiredStringProperty(hive.krb.keytab);String krbPrincipal PropertiesUtil.getRequiredStringProperty(hive.krb.principal);if (StringUtils.isEmpty(krbConf) || StringUtils.isEmpty(krbKeytab) || StringUtils.isEmpty(krbPrincipal)){throw new RuntimeException(------------------------Kerberos认证文件不存在--------------------------);}//java 程序本身自带kerberos客户端需要krbConf. 可以进行当前节点的kerberos认证System.setProperty(java.security.krb5.conf,krbConf);Configuration configuration new Configuration();configuration.set(hadoop.security.authorization,kerberos);//指定keytab文件和principal为当前java程序配置认证configuration.set(keytab.file,krbKeytab);configuration.setBoolean(hadoop.security.authorization,true);configuration.set(kerberos.principal,krbPrincipal) ;try {UserGroupInformation.setConfiguration(configuration);UserGroupInformation.loginUserFromKeytab(krbPrincipal,krbKeytab);}catch (IOException ioe){System.err.println(ioe.getMessage());}}}
properties配置文件
conf.properties示例
krb.conf/etc/krb5.conf
hive.krb.key/opt/keytabs/hive.keytab
hive.krb.principalhiveExample.COM
