How to set configuration in Hive-Site.xml file for hive metastore connection?

Question

I want to connect MetaStore using the java code. I have no idea how to set configuration setting in Hive-Site.xml file and where I'll post the Hive-Site.xml file. Please help.

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;

public class HiveMetastoreJDBCTest {

    public static void main(String[] args) throws Exception {

        Connection conn = null;
        try {
            HiveConf conf = new HiveConf();
            conf.addResource(new Path("file:///path/to/hive-site.xml"));
            Class.forName(conf.getVar(ConfVars.METASTORE_CONNECTION_DRIVER));
            conn = DriverManager.getConnection(
                    conf.getVar(ConfVars.METASTORECONNECTURLKEY),
                    conf.getVar(ConfVars.METASTORE_CONNECTION_USER_NAME),
                    conf.getVar(ConfVars.METASTOREPWD));

            Statement st = conn.createStatement();
            ResultSet rs = st.executeQuery(
                "select t.tbl_name, s.location from tbls t " +
                "join sds s on t.sd_id = s.sd_id");
            while (rs.next()) {
                System.out.println(rs.getString(1) + " : " + rs.getString(2));
            }
        }        

    }
}

Solution 1

Add these lines in your hive-site.xml:

<property>
  <name>hive.metastore.local</name>
  <value>true</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionURL</name>
  <value>jdbc:mysql://localhost:3306/hive</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>hiveuser</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>hivepass</value>
</property>

In jdbc:mysql://localhost:3306/hive, 3306 is your default mysql port; hive is our mysql database name for hive metastore. Change hiveuser to your mysql hive username and hivepass to your mysql hive password.

Do this step in terminal, if you haven't created a database for hive metastore in mysql:

mysql -u root -p

Enter your mysql root password.

mysql> create database hive;

mysql> create user 'hiveuser'@'%' IDENTIFIED BY 'hivepass';

mysql> GRANT all on *.* to 'hiveuser'@localhost identified by 'hivepass';

mysql> flush privileges;

Here, hiveuser and hivepass are whatever username and password you give for hive metastore respectively.

NOTE: You need to have mysql-jdbc-connector.jar in $HIVE_HOME/lib and $HADOOP_HOME/lib

Solution 2

Regarding Hive-site.xml here is sample from my testing machine. This is for setting up hive metastore with MySql server installed on localhost.

<configuration>
<property>
 <name>javax.jdo.option.ConnectionURL</name>
 <value>jdbc:mysql://localhost/metastore?createDatabaseIfNotExist=true</value>
 <description>metadata is stored in a MySQL server</description>
</property>
<property>
 <name>javax.jdo.option.ConnectionDriverName</name>
 <value>com.mysql.jdbc.Driver</value>
 <description>MySQL JDBC driver class</description>
</property>
<property>
 <name>javax.jdo.option.ConnectionUserName</name>
 <value>hive</value>
 <description>user name for connecting to mysql server </description>
</property>
<property>
 <name>javax.jdo.option.ConnectionPassword</name>
 <value>123456</value>
 <description>password for connecting to mysql server </description>
</property>
</configuration>

This file you need to place inside <system_path>/apache-hive-x.xx.x-bin/conf directory

I do not have much idea about how to use this file in java. But by specifying connection string in java code you can do it as below

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

public class WriteToHive {
    private static String driverName = "org.apache.hadoop.hive.jdbc.HiveDriver";
    static Connection con;
    static Statement stmt;

    public WriteToHive() throws SQLException, ClassNotFoundException, Exception {
        try {
            Class.forName(driverName);
        } catch (ClassNotFoundException e){
            e.printStackTrace();
            throw new ClassNotFoundException("No JDBC Hive Driver found");
            //System.exit(1);
        } catch (Exception e) {
            e.printStackTrace();
            throw new Exception(e);
            //System.exit(1);
        }

        con = DriverManager.getConnection("jdbc:hive://localhost:10000/rajen","","");
        stmt = con.createStatement();
    }

    public static void main(String[] args) throws SQLException {
        try {
            Class.forName(driverName);
        } catch (ClassNotFoundException e){
            e.printStackTrace();
            System.exit(1);
        } catch (Exception e) {
            e.printStackTrace();
            System.exit(1);
        }
        con = DriverManager.getConnection("jdbc:hive://localhost:10000/rajen","","");
        stmt = con.createStatement();
        //Connection con = DriverManager.getConnection("jdbc:hive://","","");
        String tableName = "company_mas_hive_eclipse_trial";

        ResultSet res = stmt.executeQuery("use rajen");

        String sql = "DROP TABLE IF EXISTS " + tableName;
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);

        sql = "CREATE TABLE IF NOT EXISTS rajen.company_mas_hive_eclipse_trial (" +
              "Name string," + 
              "dateofincorporation string," + 
              "country string)" +
              "ROW FORMAT DELIMITED FIELDS TERMINATED BY \",\"";
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);

        sql = "show tables '" + tableName + "'";
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);

        if (res.next()){
            System.out.println(res.getString(1));
        }

        sql = "describe " + tableName;
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);
        System.out.println("=========================================");
        while (res.next()) {
          System.out.println(res.getString(1) + "\t" + res.getString(2));
        }
        System.out.println("=========================================");

        // load data into table
        // NOTE: filepath has to be local to the hive server
        // NOTE: /tmp/a.txt is a ctrl-A separated file with two fields per line
        String filepath = "/home/seo/Refrence_Doc/sampledata/companymas"; //"/rajen/companymas";
        sql = "load data local inpath '" + filepath + "' into table " + tableName;
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);

        // load data into table
        // NOTE: filepath has to be local to the hive server
        // NOTE: /tmp/a.txt is a ctrl-A separated file with two fields per line
        filepath = "/rajen/companymas";
        sql = "load data inpath '" + filepath + "' into table " + tableName;
        System.out.println("Running: " + sql);
        //res = stmt.executeQuery(sql);

        // select * query
        sql = "select * from " + tableName;
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);
        while (res.next()) {
            System.out.println(String.valueOf(res.getString(1)) + "\t" + res.getString(2));
        }

        // regular hive query
        sql = "select count(*) from " + tableName;
        System.out.println("Running: " + sql);
        res = stmt.executeQuery(sql);
        while (res.next()) {
            System.out.println(res.getString(1));
        }
    }

    public void createTable(String def, String dbname) throws SQLException{
        @SuppressWarnings("unused")
        ResultSet res = stmt.executeQuery("use " + dbname);
        stmt.executeQuery(def);
    }

    public static void loadData(String filepath, String tableName) throws SQLException{
        stmt.executeQuery("load data local inpath '" + filepath + "' into table " + tableName);
    }
}

Comments

[mohit sharma]

Thanks for valuable reply. But I am trying to connect Hive-Metastore using the java code not MySql. If you have the code please give me.