Attachment 'TestOwlDriverSample.java'

Download

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 
  19 
  20 import java.util.ArrayList;
  21 import java.util.List;
  22 
  23 import org.apache.hadoop.owl.common.ErrorType;
  24 import org.apache.hadoop.owl.common.LogHandler;
  25 import org.apache.hadoop.owl.common.OwlException;
  26 import org.apache.hadoop.owl.driver.OwlDriver;
  27 import org.apache.hadoop.owl.driver.OwlTableBuilder;
  28 import org.apache.hadoop.owl.protocol.ColumnType;
  29 import org.apache.hadoop.owl.protocol.OwlColumnSchema;
  30 import org.apache.hadoop.owl.protocol.OwlDatabase;
  31 import org.apache.hadoop.owl.protocol.OwlKeyValue;
  32 import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
  33 import org.apache.hadoop.owl.protocol.OwlPartition;
  34 import org.apache.hadoop.owl.protocol.OwlSchema;
  35 import org.apache.hadoop.owl.protocol.OwlTable;
  36 import org.apache.hadoop.owl.protocol.OwlTableName;
  37 import org.apache.hadoop.owl.protocol.OwlKey.DataType;
  38 
  39 //Test program with sample code for working with owl metadata system 
  40 public class TestOwlDriverSample {
  41 
  42     private static String owlUrl = "http://localhost:8080/owl/rest";
  43 
  44     private static final String owlDatabaseName = "testSampleDB";
  45     private static final String owlTableName = "testSampleTable";
  46 
  47     private OwlTableName tableName = new OwlTableName(owlDatabaseName, owlTableName);
  48     
  49     private OwlSchema schema;
  50     private OwlLoaderInfo loaderInfo;
  51 
  52     private OwlDriver driver;
  53 
  54     //Initialize the owl driver, cleanup existing owldatabase/owltable, create owldatabase and owltable,
  55     //publish partitions into the owl table
  56     public void initialize() throws OwlException {
  57 
  58         
  59         schema = new OwlSchema();
  60         schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
  61         schema.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING));
  62         schema.addColumnSchema(new OwlColumnSchema("c3", ColumnType.LONG));
  63         
  64         loaderInfo = new OwlLoaderInfo("org.apache.hadoop.owl.testdriver.SampleInputStorageDriverImpl");
  65         
  66         cleanup(true);
  67         createOwlDatabase();
  68         createOwlTable();
  69         
  70         publishData(1, "test1", 11, "hdfs://localhost:9000/data/1/test1/11");
  71         publishData(1, "test1", 12, "hdfs://localhost:9000/data/1/test1/12");
  72         publishData(1, "test2", 11, "hdfs://localhost:9000/data/1/test2/11");
  73         publishData(1, "test2", 13, "hdfs://localhost:9000/data/1/test2/13");
  74         publishData(2, "test1", 12, "hdfs://localhost:9000/data/2/test1/12");
  75         publishData(2, "test2", 11, "hdfs://localhost:9000/data/2/test2/11");
  76         publishData(2, "test2", 13, "hdfs://localhost:9000/data/2/test2/13");
  77     }
  78     
  79     //drop existing table and database
  80     public void cleanup(boolean ignoreAbsence) throws OwlException {
  81         try {
  82             driver.dropOwlTable(new OwlTable(tableName));
  83         } catch (OwlException oe){
  84             if ( oe.getErrorType().equals(ErrorType.ERROR_UNKNOWN_DATABASE) || 
  85                     oe.getErrorType().equals(ErrorType.ERROR_UNKNOWN_OWLTABLE) ){
  86                 if( ! ignoreAbsence ) {
  87                     throw oe;
  88                 }
  89             } else {
  90                 throw oe;
  91             }
  92         }
  93         try {
  94             driver.dropOwlDatabase(new OwlDatabase(owlDatabaseName, "hdfs://localhost:9000/data/testdatabase"));
  95         } catch (OwlException oe){
  96             if ( oe.getErrorType().equals(ErrorType.ERROR_UNKNOWN_DATABASE) ){
  97                 if( ! ignoreAbsence ) {
  98                     throw oe;
  99                 }
 100             } else {
 101                 throw oe;
 102             }
 103         }
 104     }
 105     
 106     //Create the owl database
 107     public void createOwlDatabase() throws OwlException {
 108         OwlDatabase owlDatabase = new OwlDatabase(owlDatabaseName, "hdfs://localhost:9000/data/testdatabase");
 109         driver.createOwlDatabase(owlDatabase);
 110     }
 111     
 112     //Create the owl table
 113     public void createOwlTable() throws OwlException {
 114         OwlTable owlTable = new OwlTableBuilder().
 115             setName(tableName).
 116             addPartition("part1", DataType.INT).
 117             addPartition("part2", DataType.STRING).
 118             addPartition("part3", DataType.INT).
 119             setSchema(schema).
 120             build();
 121         
 122         driver.createOwlTable(owlTable);
 123     }
 124     
 125     //Publish partition with given values
 126     void publishData(int part1Value, String part2Value, int part3Value, String location) throws OwlException {
 127 
 128         List<OwlKeyValue> keyValues = new ArrayList<OwlKeyValue>();
 129         
 130         keyValues.add(new OwlKeyValue("part1", Integer.valueOf(part1Value)));
 131         keyValues.add(new OwlKeyValue("part2", part2Value));
 132         keyValues.add(new OwlKeyValue("part3", Integer.valueOf(part3Value)));
 133 
 134         driver.publish(tableName, keyValues, null, location, schema, loaderInfo);
 135     }
 136     
 137     //Select partition for the given filter
 138     public void selectPartitions(String filter, int expectedPartitions) throws OwlException {
 139         List<OwlPartition> partitions = driver.getPartitions(tableName, filter);
 140         
 141         System.out.println("Filter \"" + filter + "\" matched " + partitions.size() + " partition(s).");
 142         for(OwlPartition partition : partitions) {
 143             System.out.println("\t Partition location " + partition.getStorageLocation());
 144         }
 145         
 146     }
 147     
 148     public void testSample() throws OwlException {
 149         System.out.println("Running test with owl url " + owlUrl);
 150         LogHandler.configureNullLogAppender();
 151 
 152         driver = new OwlDriver(owlUrl);
 153 
 154         cleanup(true);
 155         initialize();
 156         selectPartitions("part1 = 1 and part2 = \"test2\"", 2);
 157         selectPartitions("part3 = 11 or part3 = 12", 5);
 158         cleanup(false);
 159     }
 160     
 161     public static void main(String[] args) throws OwlException {
 162         if( args.length == 1 ) {
 163             owlUrl = args[0];
 164         }
 165         
 166         new TestOwlDriverSample().testSample();
 167     }
 168 }

Attached Files

To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.

You are not allowed to attach a file to this page.