HIVE-7209 : allow metastore authorization api calls to be restricted …

…to certain invokers (Thejas Nair via Sushanth Sowmyan, Ashutosh Chauhan)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1607753 13f79535-47bb-0310-9956-ffa450edef68

conf/hive-default.xml.template

@@ -1624,8 +1624,9 @@
 <property>
   <name>hive.security.metastore.authorization.manager</name>
   <value>org.apache.hadoop.hive.ql.security.authorization.DefaultHiveMetastoreAuthorizationProvider</value>
-  <description>authorization manager class name to be used in the metastore for authorization.
-  The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider. 
+  <description>Names of authorization manager classes (comma separated) to be used in the metastore for authorization.
+  The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider.
+  All authorization manager classes have to successfully authorize the metastore api call for the command execution to be allowed.
   </description>
 </property>
 

itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java

@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.Test;
+
+/**
+ * Test case for {@link MetaStoreAuthzAPIAuthorizerEmbedOnly} The authorizer is
+ * supposed to allow api calls for metastore in embedded mode while disallowing
+ * them in remote metastore mode. Note that this is an abstract class, the
+ * subclasses that set the mode and the tests here get run as part of their
+ * testing.
+ */
+public abstract class TestAuthorizationApiAuthorizer {
+  protected static boolean isRemoteMetastoreMode;
+  private static HiveConf hiveConf;
+  private static HiveMetaStoreClient msc;
+
+  protected static void setup() throws Exception {
+    System.err.println("Running with remoteMode = " + isRemoteMetastoreMode);
+    System.setProperty("hive.metastore.pre.event.listeners",
+        AuthorizationPreEventListener.class.getName());
+    System.setProperty("hive.security.metastore.authorization.manager",
+        MetaStoreAuthzAPIAuthorizerEmbedOnly.class.getName());
+
+    hiveConf = new HiveConf();
+    if (isRemoteMetastoreMode) {
+      int port = MetaStoreUtils.findFreePort();
+      MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+      hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+    }
+    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+
+    msc = new HiveMetaStoreClient(hiveConf, null);
+
+  }
+
+  interface FunctionInvoker {
+    public void invoke() throws Exception;
+  }
+
+  /**
+   * Test the if authorization failed/passed for FunctionInvoker that invokes a metastore client
+   * api call
+   * @param mscFunctionInvoker
+   * @throws Exception
+   */
+  private void testFunction(FunctionInvoker mscFunctionInvoker) throws Exception {
+    boolean caughtEx = false;
+    try {
+      try {
+        mscFunctionInvoker.invoke();
+      } catch (RuntimeException e) {
+        // A hack to verify that authorization check passed. Exception can be thrown be cause
+        // the functions are not being called with valid params.
+        // verify that exception has come from ObjectStore code, which means that the
+        // authorization checks passed.
+        String exStackString = ExceptionUtils.getStackTrace(e);
+        assertTrue("Verifying this exception came after authorization check",
+            exStackString.contains("org.apache.hadoop.hive.metastore.ObjectStore"));
+        // If its not an exception caused by auth check, ignore it
+      }
+      assertFalse("Authz Exception should have been thrown in remote mode", isRemoteMetastoreMode);
+      System.err.println("No auth exception thrown");
+    } catch (MetaException e) {
+      System.err.println("Caught exception");
+      caughtEx = true;
+      assertTrue(e.getMessage().contains(MetaStoreAuthzAPIAuthorizerEmbedOnly.errMsg));
+    }
+    if (!isRemoteMetastoreMode) {
+      assertFalse("No exception should be thrown in embedded mode", caughtEx);
+    }
+  }
+
+  @Test
+  public void testGrantPriv() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.grant_privileges(new PrivilegeBag(new ArrayList<HiveObjectPrivilege>()));
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testRevokePriv() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.revoke_privileges(new PrivilegeBag(new ArrayList<HiveObjectPrivilege>()));
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testGrantRole() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.grant_role(null, null, null, null, null, true);
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testRevokeRole() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.revoke_role(null, null, null);
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testCreateRole() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.create_role(new Role());
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testDropRole() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.drop_role(null);
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testListRoles() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.list_roles(null, null);
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testGetPrivSet() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.get_privilege_set(new HiveObjectRef(), null, new ArrayList<String>());
+      }
+    };
+    testFunction(invoker);
+  }
+
+  @Test
+  public void testListPriv() throws Exception {
+    FunctionInvoker invoker = new FunctionInvoker() {
+      @Override
+      public void invoke() throws Exception {
+        msc.list_privileges(null, PrincipalType.USER, new HiveObjectRef());
+      }
+    };
+    testFunction(invoker);
+  }
+
+
+
+}

itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java

@@ -0,0 +1,16 @@
+package org.apache.hadoop.hive.metastore;
+
+import org.junit.BeforeClass;
+
+/**
+ * Test {@link TestAuthorizationApiAuthorizer} in embedded mode of metastore
+ */
+public class TestAuthzApiEmbedAuthorizerInEmbed extends TestAuthorizationApiAuthorizer {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    isRemoteMetastoreMode = false; // embedded metastore mode
+    TestAuthorizationApiAuthorizer.setup();
+  }
+
+}

itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java

@@ -0,0 +1,16 @@
+package org.apache.hadoop.hive.metastore;
+
+import org.junit.BeforeClass;
+
+/**
+ * Test {@link TestAuthorizationApiAuthorizer} in remote mode of metastore
+ */
+public class TestAuthzApiEmbedAuthorizerInRemote extends TestAuthorizationApiAuthorizer {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    isRemoteMetastoreMode = true; // remote metastore mode
+    TestAuthorizationApiAuthorizer.setup();
+  }
+
+}

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java

@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.hive.ql.security;
 
-import java.io.IOException;
-import java.net.ServerSocket;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -81,14 +79,6 @@ protected void setUp() throws Exception {
     driver = new Driver(clientHiveConf);
   }
 
-  private static String getFreeAvailablePort() throws IOException {
-    ServerSocket socket = new ServerSocket(0);
-    socket.setReuseAddress(true);
-    int port = socket.getLocalPort();
-    socket.close();
-    return "" + port;
-  }
-
   @Override
   protected void tearDown() throws Exception {
     super.tearDown();

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java

@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.security;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.security.DummyHiveMetastoreAuthorizationProvider.AuthCallContext;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test case for verifying that multiple
+ * {@link org.apache.hadoop.hive.metastore.AuthorizationPreEventListener}s can
+ * be set and they get called.
+ */
+public class TestMultiAuthorizationPreEventListener {
+  private static HiveConf clientHiveConf;
+  private static HiveMetaStoreClient msc;
+  private static Driver driver;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+
+
+    int port = MetaStoreUtils.findFreePort();
+
+    System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
+        AuthorizationPreEventListener.class.getName());
+
+    // Set two dummy classes as authorizatin managers. Two instances should get created.
+    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname,
+        DummyHiveMetastoreAuthorizationProvider.class.getName() + ","
+            + DummyHiveMetastoreAuthorizationProvider.class.getName());
+
+    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
+        HadoopDefaultMetastoreAuthenticator.class.getName());
+
+    MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+
+    clientHiveConf = new HiveConf();
+
+    clientHiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+    clientHiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+
+    SessionState.start(new CliSessionState(clientHiveConf));
+    msc = new HiveMetaStoreClient(clientHiveConf, null);
+    driver = new Driver(clientHiveConf);
+  }
+
+  @Test
+  public void testMultipleAuthorizationListners() throws Exception {
+    String dbName = "hive" + this.getClass().getSimpleName().toLowerCase();
+    List<AuthCallContext> authCalls = DummyHiveMetastoreAuthorizationProvider.authCalls;
+    int listSize = 0;
+    assertEquals(listSize, authCalls.size());
+
+    driver.run("create database " + dbName);
+    // verify that there are two calls because of two instances of the authorization provider
+    listSize = 2;
+    assertEquals(listSize, authCalls.size());
+
+    // verify that the actual action also went through
+    Database db = msc.getDatabase(dbName);
+    Database dbFromEvent = (Database)assertAndExtractSingleObjectFromEvent(listSize, authCalls,
+        DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.DB);
+    validateCreateDb(db,dbFromEvent);
+  }
+
+  public Object assertAndExtractSingleObjectFromEvent(int listSize,
+      List<AuthCallContext> authCalls,
+      DummyHiveMetastoreAuthorizationProvider.AuthCallContextType callType) {
+    assertEquals(listSize, authCalls.size());
+    assertEquals(1,authCalls.get(listSize-1).authObjects.size());
+
+    assertEquals(callType,authCalls.get(listSize-1).type);
+    return (authCalls.get(listSize-1).authObjects.get(0));
+  }
+
+
+  private void validateCreateDb(Database expectedDb, Database actualDb) {
+    assertEquals(expectedDb.getName(), actualDb.getName());
+    assertEquals(expectedDb.getLocationUri(), actualDb.getLocationUri());
+  }
+
+
+}