1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.http;
19
20 import static org.mockito.Mockito.doReturn;
21 import static org.mockito.Mockito.mock;
22
23 import java.io.IOException;
24 import java.io.PrintWriter;
25 import java.net.HttpURLConnection;
26 import java.net.URI;
27 import java.net.URL;
28 import java.util.Arrays;
29 import java.util.Enumeration;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.SortedSet;
34 import java.util.TreeSet;
35 import java.util.concurrent.CountDownLatch;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.Executors;
38
39 import javax.servlet.Filter;
40 import javax.servlet.FilterChain;
41 import javax.servlet.FilterConfig;
42 import javax.servlet.ServletContext;
43 import javax.servlet.ServletException;
44 import javax.servlet.ServletRequest;
45 import javax.servlet.ServletResponse;
46 import javax.servlet.http.HttpServlet;
47 import javax.servlet.http.HttpServletRequest;
48 import javax.servlet.http.HttpServletRequestWrapper;
49 import javax.servlet.http.HttpServletResponse;
50
51 import junit.framework.Assert;
52
53 import org.apache.commons.logging.Log;
54 import org.apache.commons.logging.LogFactory;
55 import org.apache.hadoop.conf.Configuration;
56 import org.apache.hadoop.fs.CommonConfigurationKeys;
57 import org.apache.hadoop.hbase.testclassification.MiscTests;
58 import org.apache.hadoop.hbase.testclassification.SmallTests;
59 import org.apache.hadoop.hbase.http.HttpServer.QuotingInputFilter.RequestQuoter;
60 import org.apache.hadoop.hbase.http.resource.JerseyResource;
61 import org.apache.hadoop.net.NetUtils;
62 import org.apache.hadoop.security.Groups;
63 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
64 import org.apache.hadoop.security.UserGroupInformation;
65 import org.apache.hadoop.security.authorize.AccessControlList;
66 import org.junit.AfterClass;
67 import org.junit.BeforeClass;
68 import org.junit.Ignore;
69 import org.junit.Test;
70 import org.junit.experimental.categories.Category;
71 import org.mockito.Mockito;
72 import org.mockito.internal.util.reflection.Whitebox;
73 import org.mortbay.jetty.Connector;
74 import org.mortbay.util.ajax.JSON;
75
76 @Category({MiscTests.class, SmallTests.class})
77 public class TestHttpServer extends HttpServerFunctionalTest {
78 private static final Log LOG = LogFactory.getLog(TestHttpServer.class);
79 private static HttpServer server;
80 private static URL baseUrl;
81 private static final int MAX_THREADS = 10;
82
83 @SuppressWarnings("serial")
84 public static class EchoMapServlet extends HttpServlet {
85 @SuppressWarnings("unchecked")
86 @Override
87 public void doGet(HttpServletRequest request,
88 HttpServletResponse response
89 ) throws ServletException, IOException {
90 PrintWriter out = response.getWriter();
91 Map<String, String[]> params = request.getParameterMap();
92 SortedSet<String> keys = new TreeSet<String>(params.keySet());
93 for(String key: keys) {
94 out.print(key);
95 out.print(':');
96 String[] values = params.get(key);
97 if (values.length > 0) {
98 out.print(values[0]);
99 for(int i=1; i < values.length; ++i) {
100 out.print(',');
101 out.print(values[i]);
102 }
103 }
104 out.print('\n');
105 }
106 out.close();
107 }
108 }
109
110 @SuppressWarnings("serial")
111 public static class EchoServlet extends HttpServlet {
112 @SuppressWarnings("unchecked")
113 @Override
114 public void doGet(HttpServletRequest request,
115 HttpServletResponse response
116 ) throws ServletException, IOException {
117 PrintWriter out = response.getWriter();
118 SortedSet<String> sortedKeys = new TreeSet<String>();
119 Enumeration<String> keys = request.getParameterNames();
120 while(keys.hasMoreElements()) {
121 sortedKeys.add(keys.nextElement());
122 }
123 for(String key: sortedKeys) {
124 out.print(key);
125 out.print(':');
126 out.print(request.getParameter(key));
127 out.print('\n');
128 }
129 out.close();
130 }
131 }
132
133 @SuppressWarnings("serial")
134 public static class LongHeaderServlet extends HttpServlet {
135 @Override
136 public void doGet(HttpServletRequest request,
137 HttpServletResponse response
138 ) throws ServletException, IOException {
139 Assert.assertEquals(63 * 1024, request.getHeader("longheader").length());
140 response.setStatus(HttpServletResponse.SC_OK);
141 }
142 }
143
144 @SuppressWarnings("serial")
145 public static class HtmlContentServlet extends HttpServlet {
146 @Override
147 public void doGet(HttpServletRequest request,
148 HttpServletResponse response
149 ) throws ServletException, IOException {
150 response.setContentType("text/html");
151 PrintWriter out = response.getWriter();
152 out.print("hello world");
153 out.close();
154 }
155 }
156
157 @BeforeClass public static void setup() throws Exception {
158 Configuration conf = new Configuration();
159 conf.setInt(HttpServer.HTTP_MAX_THREADS, 10);
160 server = createTestServer(conf);
161 server.addServlet("echo", "/echo", EchoServlet.class);
162 server.addServlet("echomap", "/echomap", EchoMapServlet.class);
163 server.addServlet("htmlcontent", "/htmlcontent", HtmlContentServlet.class);
164 server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
165 server.addJerseyResourcePackage(
166 JerseyResource.class.getPackage().getName(), "/jersey/*");
167 server.start();
168 baseUrl = getServerURL(server);
169 LOG.info("HTTP server started: "+ baseUrl);
170 }
171
172 @AfterClass public static void cleanup() throws Exception {
173 server.stop();
174 }
175
176
177 @Test public void testMaxThreads() throws Exception {
178 int clientThreads = MAX_THREADS * 10;
179 Executor executor = Executors.newFixedThreadPool(clientThreads);
180
181 final CountDownLatch ready = new CountDownLatch(clientThreads);
182 final CountDownLatch start = new CountDownLatch(1);
183 for (int i = 0; i < clientThreads; i++) {
184 executor.execute(new Runnable() {
185 @Override
186 public void run() {
187 ready.countDown();
188 try {
189 start.await();
190 assertEquals("a:b\nc:d\n",
191 readOutput(new URL(baseUrl, "/echo?a=b&c=d")));
192 int serverThreads = server.webServer.getThreadPool().getThreads();
193 assertTrue("More threads are started than expected, Server Threads count: "
194 + serverThreads, serverThreads <= MAX_THREADS);
195 System.out.println("Number of threads = " + serverThreads +
196 " which is less or equal than the max = " + MAX_THREADS);
197 } catch (Exception e) {
198
199 }
200 }
201 });
202 }
203
204 ready.await();
205 start.countDown();
206 }
207
208 @Test public void testEcho() throws Exception {
209 assertEquals("a:b\nc:d\n",
210 readOutput(new URL(baseUrl, "/echo?a=b&c=d")));
211 assertEquals("a:b\nc<:d\ne:>\n",
212 readOutput(new URL(baseUrl, "/echo?a=b&c<=d&e=>")));
213 }
214
215
216 @Test public void testEchoMap() throws Exception {
217 assertEquals("a:b\nc:d\n",
218 readOutput(new URL(baseUrl, "/echomap?a=b&c=d")));
219 assertEquals("a:b,>\nc<:d\n",
220 readOutput(new URL(baseUrl, "/echomap?a=b&c<=d&a=>")));
221 }
222
223
224
225
226
227
228 @Test public void testLongHeader() throws Exception {
229 URL url = new URL(baseUrl, "/longheader");
230 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
231 StringBuilder sb = new StringBuilder();
232 for (int i = 0 ; i < 63 * 1024; i++) {
233 sb.append("a");
234 }
235 conn.setRequestProperty("longheader", sb.toString());
236 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
237 }
238
239 @Test
240 @Ignore
241 public void testContentTypes() throws Exception {
242
243 URL cssUrl = new URL(baseUrl, "/static/test.css");
244 HttpURLConnection conn = (HttpURLConnection)cssUrl.openConnection();
245 conn.connect();
246 assertEquals(200, conn.getResponseCode());
247 assertEquals("text/css", conn.getContentType());
248
249
250 URL servletUrl = new URL(baseUrl, "/echo?a=b");
251 conn = (HttpURLConnection)servletUrl.openConnection();
252 conn.connect();
253 assertEquals(200, conn.getResponseCode());
254 assertEquals("text/plain; charset=utf-8", conn.getContentType());
255
256
257
258 servletUrl = new URL(baseUrl, "/echo?a=b.css");
259 conn = (HttpURLConnection)servletUrl.openConnection();
260 conn.connect();
261 assertEquals(200, conn.getResponseCode());
262 assertEquals("text/plain; charset=utf-8", conn.getContentType());
263
264
265 servletUrl = new URL(baseUrl, "/htmlcontent");
266 conn = (HttpURLConnection)servletUrl.openConnection();
267 conn.connect();
268 assertEquals(200, conn.getResponseCode());
269 assertEquals("text/html; charset=utf-8", conn.getContentType());
270
271
272 servletUrl = new URL(baseUrl, "/testjsp.jsp");
273 conn = (HttpURLConnection)servletUrl.openConnection();
274 conn.connect();
275 assertEquals(200, conn.getResponseCode());
276 assertEquals("text/html; charset=utf-8", conn.getContentType());
277 }
278
279
280
281
282
283
284
285 public static class DummyServletFilter implements Filter {
286 @Override
287 public void destroy() { }
288
289 @Override
290 public void doFilter(ServletRequest request, ServletResponse response,
291 FilterChain filterChain) throws IOException, ServletException {
292 final String userName = request.getParameter("user.name");
293 ServletRequest requestModified =
294 new HttpServletRequestWrapper((HttpServletRequest) request) {
295 @Override
296 public String getRemoteUser() {
297 return userName;
298 }
299 };
300 filterChain.doFilter(requestModified, response);
301 }
302
303 @Override
304 public void init(FilterConfig arg0) throws ServletException { }
305 }
306
307
308
309
310
311 public static class DummyFilterInitializer extends FilterInitializer {
312 public DummyFilterInitializer() {
313 }
314
315 @Override
316 public void initFilter(FilterContainer container, Configuration conf) {
317 container.addFilter("DummyFilter", DummyServletFilter.class.getName(), null);
318 }
319 }
320
321
322
323
324
325
326
327
328
329
330
331 static int getHttpStatusCode(String urlstring, String userName)
332 throws IOException {
333 URL url = new URL(urlstring + "?user.name=" + userName);
334 System.out.println("Accessing " + url + " as user " + userName);
335 HttpURLConnection connection = (HttpURLConnection)url.openConnection();
336 connection.connect();
337 return connection.getResponseCode();
338 }
339
340
341
342
343 public static class MyGroupsProvider extends ShellBasedUnixGroupsMapping {
344 static Map<String, List<String>> mapping = new HashMap<String, List<String>>();
345
346 static void clearMapping() {
347 mapping.clear();
348 }
349
350 @Override
351 public List<String> getGroups(String user) throws IOException {
352 return mapping.get(user);
353 }
354 }
355
356
357
358
359
360
361
362 @Test
363 @Ignore
364 public void testDisabledAuthorizationOfDefaultServlets() throws Exception {
365
366 Configuration conf = new Configuration();
367
368
369 conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY,
370 DummyFilterInitializer.class.getName());
371 conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
372 MyGroupsProvider.class.getName());
373 Groups.getUserToGroupsMappingService(conf);
374 MyGroupsProvider.clearMapping();
375 MyGroupsProvider.mapping.put("userA", Arrays.asList("groupA"));
376 MyGroupsProvider.mapping.put("userB", Arrays.asList("groupB"));
377
378 HttpServer myServer = new HttpServer.Builder().setName("test")
379 .addEndpoint(new URI("http://localhost:0")).setFindPort(true).build();
380 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
381 myServer.start();
382 String serverURL = "http://" + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
383 for (String servlet : new String[] { "conf", "logs", "stacks",
384 "logLevel", "metrics" }) {
385 for (String user : new String[] { "userA", "userB" }) {
386 assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(serverURL
387 + servlet, user));
388 }
389 }
390 myServer.stop();
391 }
392
393
394
395
396
397
398
399 @Test
400 @Ignore
401 public void testAuthorizationOfDefaultServlets() throws Exception {
402 Configuration conf = new Configuration();
403 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
404 true);
405 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN,
406 true);
407 conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY,
408 DummyFilterInitializer.class.getName());
409
410 conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
411 MyGroupsProvider.class.getName());
412 Groups.getUserToGroupsMappingService(conf);
413 MyGroupsProvider.clearMapping();
414 MyGroupsProvider.mapping.put("userA", Arrays.asList("groupA"));
415 MyGroupsProvider.mapping.put("userB", Arrays.asList("groupB"));
416 MyGroupsProvider.mapping.put("userC", Arrays.asList("groupC"));
417 MyGroupsProvider.mapping.put("userD", Arrays.asList("groupD"));
418 MyGroupsProvider.mapping.put("userE", Arrays.asList("groupE"));
419
420 HttpServer myServer = new HttpServer.Builder().setName("test")
421 .addEndpoint(new URI("http://localhost:0")).setFindPort(true).setConf(conf)
422 .setACL(new AccessControlList("userA,userB groupC,groupD")).build();
423 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
424 myServer.start();
425
426 String serverURL = "http://"
427 + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
428 for (String servlet : new String[] { "conf", "logs", "stacks",
429 "logLevel", "metrics" }) {
430 for (String user : new String[] { "userA", "userB", "userC", "userD" }) {
431 assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(serverURL
432 + servlet, user));
433 }
434 assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(
435 serverURL + servlet, "userE"));
436 }
437 myServer.stop();
438 }
439
440 @Test
441 public void testRequestQuoterWithNull() throws Exception {
442 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
443 Mockito.doReturn(null).when(request).getParameterValues("dummy");
444 RequestQuoter requestQuoter = new RequestQuoter(request);
445 String[] parameterValues = requestQuoter.getParameterValues("dummy");
446 Assert.assertEquals("It should return null "
447 + "when there are no values for the parameter", null, parameterValues);
448 }
449
450 @Test
451 public void testRequestQuoterWithNotNull() throws Exception {
452 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
453 String[] values = new String[] { "abc", "def" };
454 Mockito.doReturn(values).when(request).getParameterValues("dummy");
455 RequestQuoter requestQuoter = new RequestQuoter(request);
456 String[] parameterValues = requestQuoter.getParameterValues("dummy");
457 Assert.assertTrue("It should return Parameter Values", Arrays.equals(
458 values, parameterValues));
459 }
460
461 @SuppressWarnings("unchecked")
462 private static Map<String, Object> parse(String jsonString) {
463 return (Map<String, Object>)JSON.parse(jsonString);
464 }
465
466 @Test public void testJersey() throws Exception {
467 LOG.info("BEGIN testJersey()");
468 final String js = readOutput(new URL(baseUrl, "/jersey/foo?op=bar"));
469 final Map<String, Object> m = parse(js);
470 LOG.info("m=" + m);
471 assertEquals("foo", m.get(JerseyResource.PATH));
472 assertEquals("bar", m.get(JerseyResource.OP));
473 LOG.info("END testJersey()");
474 }
475
476 @Test
477 public void testHasAdministratorAccess() throws Exception {
478 Configuration conf = new Configuration();
479 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
480 ServletContext context = Mockito.mock(ServletContext.class);
481 Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
482 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(null);
483 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
484 Mockito.when(request.getRemoteUser()).thenReturn(null);
485 HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
486
487
488 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
489
490
491 response = Mockito.mock(HttpServletResponse.class);
492 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
493 Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
494 Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
495
496
497 response = Mockito.mock(HttpServletResponse.class);
498 Mockito.when(request.getRemoteUser()).thenReturn("foo");
499 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
500
501
502 response = Mockito.mock(HttpServletResponse.class);
503 AccessControlList acls = Mockito.mock(AccessControlList.class);
504 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
505 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
506 Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
507 Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
508
509
510 response = Mockito.mock(HttpServletResponse.class);
511 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(true);
512 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
513 Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
514
515 }
516
517 @Test
518 public void testRequiresAuthorizationAccess() throws Exception {
519 Configuration conf = new Configuration();
520 ServletContext context = Mockito.mock(ServletContext.class);
521 Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
522 HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
523 HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
524
525
526 Assert.assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response));
527
528
529 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, true);
530 conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
531 AccessControlList acls = Mockito.mock(AccessControlList.class);
532 Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
533 Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
534 Assert.assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response));
535 }
536
537 @Test public void testBindAddress() throws Exception {
538 checkBindAddress("localhost", 0, false).stop();
539
540 HttpServer myServer = checkBindAddress("localhost", 0, false);
541 HttpServer myServer2 = null;
542 try {
543 int port = myServer.getConnectorAddress(0).getPort();
544
545 myServer2 = checkBindAddress("localhost", port, true);
546
547 port = myServer2.getConnectorAddress(0).getPort();
548 myServer2.stop();
549 assertNull(myServer2.getConnectorAddress(0));
550 myServer2.openListeners();
551 assertEquals(port, myServer2.getConnectorAddress(0).getPort());
552 } finally {
553 myServer.stop();
554 if (myServer2 != null) {
555 myServer2.stop();
556 }
557 }
558 }
559
560 private HttpServer checkBindAddress(String host, int port, boolean findPort)
561 throws Exception {
562 HttpServer server = createServer(host, port);
563 try {
564
565 List<?> listeners = (List<?>) Whitebox.getInternalState(server,
566 "listeners");
567 Connector listener = (Connector) Whitebox.getInternalState(
568 listeners.get(0), "listener");
569
570 assertEquals(port, listener.getPort());
571
572 server.openListeners();
573 assertEquals(host, server.getConnectorAddress(0).getHostName());
574
575 int boundPort = server.getConnectorAddress(0).getPort();
576 if (port == 0) {
577 assertTrue(boundPort != 0);
578 } else if (findPort) {
579 assertTrue(boundPort > port);
580
581
582 assertTrue(boundPort - port < 8);
583 }
584 } catch (Exception e) {
585 server.stop();
586 throw e;
587 }
588 return server;
589 }
590
591 @Test
592 public void testXFrameHeaderSameOrigin() throws Exception {
593 Configuration conf = new Configuration();
594 conf.set("hbase.http.filter.xframeoptions.mode", "SAMEORIGIN");
595
596 HttpServer myServer = new HttpServer.Builder().setName("test")
597 .addEndpoint(new URI("http://localhost:0"))
598 .setFindPort(true).setConf(conf).build();
599 myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
600 myServer.addServlet("echo", "/echo", EchoServlet.class);
601 myServer.start();
602
603 String serverURL = "http://"
604 + NetUtils.getHostPortString(myServer.getConnectorAddress(0));
605 URL url = new URL(new URL(serverURL), "/echo?a=b&c=d");
606 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
607 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
608 assertEquals("SAMEORIGIN", conn.getHeaderField("X-Frame-Options"));
609 myServer.stop();
610 }
611
612
613
614 @Test
615 public void testNoCacheHeader() throws Exception {
616 URL url = new URL(baseUrl, "/echo?a=b&c=d");
617 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
618 assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
619 assertEquals("no-cache", conn.getHeaderField("Cache-Control"));
620 assertEquals("no-cache", conn.getHeaderField("Pragma"));
621 assertNotNull(conn.getHeaderField("Expires"));
622 assertNotNull(conn.getHeaderField("Date"));
623 assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
624 assertEquals("DENY", conn.getHeaderField("X-Frame-Options"));
625 }
626
627
628
629
630 @Test
631 public void testHttpServerBuilderWithExternalConnector() throws Exception {
632 Connector c = mock(Connector.class);
633 doReturn("localhost").when(c).getHost();
634 HttpServer s = new HttpServer.Builder().setName("test").setConnector(c)
635 .build();
636 s.stop();
637 }
638 }