hive 加载外部类的常见方法

hive支持各种外部配置类实现功能, 比如各种hook,权限拦截等. 通用方法如下:

• 配置里写了类的名称, 然后通过Class.forName加载, 比如 Class<?> urlHookClass = Class.forName(updateData.urlHookClassName, true, JavaUtils.getClassLoader());;

• 类需要继承某个定义类, 然后可以用定义类进行转化, 在代码里使用: updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);

metastore 初始化时候的 hook

metastore初始化的时候, 调用了配置里定义的各种hook类.

METASTORE_INIT_HOOKS("hive.metastore.init.hooks", "",
        "A comma separated list of hooks to be invoked at the beginning of HMSHandler initialization. \n" +
        "An init hook is specified as the name of Java class which extends org.apache.hadoop.hive.metastore.MetaStoreInitListener."),

private ClassLoader classLoader;
private AlterHandler alterHandler;
private List<MetaStorePreEventListener> preListeners;
private List<MetaStoreEventListener> listeners;
private List<MetaStoreEventListener> transactionalListeners;
private List<MetaStoreEndFunctionListener> endFunctionListeners;
private List<MetaStoreInitListener> initListeners;

public class HiveMetaStore extends ThriftHiveMetastore {
    @Override
    public void init() throws MetaException {
      initListeners = MetaStoreUtils.getMetaStoreListeners(
          MetaStoreInitListener.class, hiveConf,
          hiveConf.getVar(HiveConf.ConfVars.METASTORE_INIT_HOOKS));
      for (MetaStoreInitListener singleInitListener: initListeners) {
          MetaStoreInitContext context = new MetaStoreInitContext();
          singleInitListener.onInit(context);
      }
      ...
    }
}

初始化一个外部定义类, 使用T listener = (T) Class.forName(listenerImpl.trim(), true, JavaUtils.getClassLoader()).getConstructor(Configuration.class).newInstance(conf);

  /**
   * create listener instances as per the configuration.
   *
   * @param clazz
   * @param conf
   * @param listenerImplList
   * @return
   * @throws MetaException
   */
  static <T> List<T> getMetaStoreListeners(Class<T> clazz,
      HiveConf conf, String listenerImplList) throws MetaException {

    List<T> listeners = new ArrayList<T>();
    listenerImplList = listenerImplList.trim();
    if (listenerImplList.equals("")) {
      return listeners;
    }

    String[] listenerImpls = listenerImplList.split(",");
    for (String listenerImpl : listenerImpls) {
      try {
        T listener = (T) Class.forName(
            listenerImpl.trim(), true, JavaUtils.getClassLoader()).getConstructor(
                Configuration.class).newInstance(conf);
        listeners.add(listener);
      } catch (InvocationTargetException ie) {
        throw new MetaException("Failed to instantiate listener named: "+
            listenerImpl + ", reason: " + ie.getCause());
      } catch (Exception e) {
        throw new MetaException("Failed to instantiate listener named: "+
            listenerImpl + ", reason: " + e);
      }
    }

    return listeners;
  }

自定义的类需要继承自这个方法, 后面初始化后才可以使用对应的方法

/**
 * This abstract class needs to be extended to  provide implementation of actions
 * that needs to be performed when HMSHandler is initialized
 */

public abstract class MetaStoreInitListener implements Configurable {

  private Configuration conf;

  public MetaStoreInitListener(Configuration config){
    this.conf = config;
  }

  public abstract void onInit(MetaStoreInitContext context) throws MetaException;

  @Override
  public Configuration getConf() {
    return this.conf;
  }

  @Override
  public void setConf(Configuration config) {
    this.conf = config;
  }
}

metastore初始化配置alter变更类

    String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",
        HiveAlterHandler.class.getName());
    alterHandler = (AlterHandler) ReflectionUtils.newInstance(MetaStoreUtils.getClass(
        alterHandlerName), hiveConf);

  @SuppressWarnings("unchecked")
  public static Class<? extends RawStore> getClass(String rawStoreClassName)
      throws MetaException {
    try {
      return (Class<? extends RawStore>)
          Class.forName(rawStoreClassName, true, JavaUtils.getClassLoader());
    } catch (ClassNotFoundException e) {
      throw new MetaException(rawStoreClassName + " class not found");
    }
  }

metastore获取url的方法

配置里写了类的名称, 然后直接加载 Class<?> urlHookClass = Class.forName(updateData.urlHookClassName, true, JavaUtils.getClassLoader());;

类需要继承某个定义类, 然后可以用定义类进行转化: updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);

    METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "",
        "Name of the hook to use for retrieving the JDO connection URL. If empty, the value in javax.jdo.option.ConnectionURL is used"),
        

  static class MetaStoreInitData {
    JDOConnectionURLHook urlHook = null;
    String urlHookClassName = "";
  }

  // Multiple threads could try to initialize at the same time.
  synchronized private static void initConnectionUrlHook(HiveConf hiveConf,
    MetaStoreInitData updateData) throws ClassNotFoundException {

    String className =
        hiveConf.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim();
    if (className.equals("")) {
      updateData.urlHookClassName = "";
      updateData.urlHook = null;
      return;
    }
    boolean urlHookChanged = !updateData.urlHookClassName.equals(className);
    if (updateData.urlHook == null || urlHookChanged) {
      updateData.urlHookClassName = className.trim();

      Class<?> urlHookClass = Class.forName(updateData.urlHookClassName, true,
          JavaUtils.getClassLoader());
      updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);
    }
    return;
  }

JavaUtils 类加载工具

package org.apache.hadoop.hive.common;

import java.io.Closeable;
import java.io.IOException;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Collection of Java class loading/reflection related utilities common across
 * Hive.
 */
public final class JavaUtils {

  private static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);

  /**
   * Standard way of getting classloader in Hive code (outside of Hadoop).
   *
   * Uses the context loader to get access to classpaths to auxiliary and jars
   * added with 'add jar' command. Falls back to current classloader.
   *
   * In Hadoop-related code, we use Configuration.getClassLoader().
   */
  public static ClassLoader getClassLoader() {
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    if (classLoader == null) {
      classLoader = JavaUtils.class.getClassLoader();
    }
    return classLoader;
  }

  public static Class loadClass(String className) throws ClassNotFoundException {
    return loadClass(className, true);
  }

  public static Class loadClass(String className, boolean init) throws ClassNotFoundException {
    return Class.forName(className, init, getClassLoader());
  }

  public static boolean closeClassLoadersTo(ClassLoader current, ClassLoader stop) {
    if (!isValidHierarchy(current, stop)) {
      return false;
    }
    for (; current != null && current != stop; current = current.getParent()) {
      try {
        closeClassLoader(current);
      } catch (IOException e) {
        String detailedMessage = current instanceof URLClassLoader ?
            Arrays.toString(((URLClassLoader) current).getURLs()) :
            "";
        LOG.info("Failed to close class loader " + current + " " + detailedMessage, e);
      }
    }
    return true;
  }

  // check before closing loaders, not to close app-classloader, etc. by mistake
  private static boolean isValidHierarchy(ClassLoader current, ClassLoader stop) {
    if (current == null || stop == null || current == stop) {
      return false;
    }
    for (; current != null && current != stop; current = current.getParent()) {
    }
    return current == stop;
  }

  public static void closeClassLoader(ClassLoader loader) throws IOException {
    if (loader instanceof Closeable) {
      ((Closeable) loader).close();
    } else {
      LOG.warn("Ignoring attempt to close class loader ({}) -- not instance of UDFClassLoader.",
        loader == null ? "mull" : loader.getClass().getSimpleName());
    }
  }

  /**
   * Utility method for ACID to normalize logging info.  Matches
   * {@link org.apache.hadoop.hive.metastore.api.LockRequest#toString()}
   */
  public static String lockIdToString(long extLockId) {
    return "lockid:" + extLockId;
  }
  /**
   * Utility method for ACID to normalize logging info.  Matches
   * {@link org.apache.hadoop.hive.metastore.api.LockResponse#toString()}
   */
  public static String txnIdToString(long txnId) {
    return "txnid:" + txnId;
  }

  public static String txnIdsToString(List<Long> txnIds) {
    return "Transactions requested to be aborted: " + txnIds.toString();
  }

  private JavaUtils() {
    // prevent instantiation
  }
}