Skip to content

Commit

Permalink
- implemented Unit test
Browse files Browse the repository at this point in the history
- created template folder inside /resources and modify build.gradle accordingly
- fix findbugs issue
  • Loading branch information
autumnust committed Jul 27, 2016
1 parent 71cb035 commit baa4219
Show file tree
Hide file tree
Showing 9 changed files with 151 additions and 69 deletions.

This file was deleted.

9 changes: 9 additions & 0 deletions gobblin-utility/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ test {
workingDir rootProject.rootDir
}

sourceSets{
test {
resources {
srcDir 'src/resources/templates'
}
}
}


ext.classification="library"


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied.
*/
package gobblin.runtime.util;
package gobblin.util;

import com.typesafe.config.Config;
import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package gobblin.util;

import com.typesafe.config.ConfigFactory;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
Expand Down Expand Up @@ -43,7 +42,6 @@
import gobblin.util.filesystem.PathAlterationListener;
import gobblin.util.filesystem.PathAlterationMonitor;
import gobblin.util.filesystem.PathAlterationObserver;
import gobblin.runtime.util.SimpleGeneralJobTemplate;


/**
Expand Down Expand Up @@ -143,7 +141,8 @@ public static Properties loadGenericJobConfig(Properties properties, Path jobCon
jobProps.putAll(ConfigurationConverter.getProperties(
new PropertiesConfiguration(new Path("file://", jobConfigPath).toUri().toURL())));

jobProps = (new SimpleGeneralJobTemplate(ConfigurationKeys.JOB_TEMPLATE_PATH)).getResolvedConfig(jobProps);
jobProps = (new SimpleGeneralJobTemplate(
jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH))).getResolvedConfigAsProperties(jobProps);

jobProps.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, jobConfigPath.toString());
return jobProps;
Expand Down Expand Up @@ -233,7 +232,8 @@ private static void loadGenericJobConfigsRecursive(List<Properties> jobConfigs,
propertiesConfiguration.load(inputStreamReader);
jobProps.putAll(ConfigurationConverter.getProperties(propertiesConfiguration));

jobProps = (new SimpleGeneralJobTemplate(ConfigurationKeys.JOB_TEMPLATE_PATH)).getResolvedConfig(jobProps);
jobProps = (new SimpleGeneralJobTemplate(
jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH))).getResolvedConfigAsProperties(jobProps);

jobProps.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, configFilePath.toString());
jobConfigs.add(jobProps);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,22 @@
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied.
*/
package gobblin.runtime.util;
package gobblin.util;

import com.typesafe.config.ConfigFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;

import java.util.Properties;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

import gobblin.util.SchedulerUtils;
import gobblin.configuration.ConfigurationKeys;
import gobblin.util.TemplateUtils;


public class SimpleGeneralJobTemplate implements JobTemplate {
Expand All @@ -43,15 +41,16 @@ public SimpleGeneralJobTemplate(String templatePath) {
LOGGER.info("Load the job configuration template : " + this.getClass().getName());
this.templatePath = templatePath;

try {
InputStream inputStream = getClass().getResourceAsStream(this.templatePath);
if (inputStream != null) {
configTemplate.load(inputStream);
} else {
throw new FileNotFoundException("Template file " + this.templatePath + " doesn't exist");
if (this.templatePath != null && this.templatePath.length() > 0) {
try (InputStream inputStream = getClass().getResourceAsStream("/" + this.templatePath)) {
if (inputStream != null) {
configTemplate.load(inputStream);
} else {
throw new FileNotFoundException("Template file " + this.templatePath + " doesn't exist");
}
} catch (IOException e) {
throw new RuntimeException("Failure to loading template files into i/o stream");
}
} catch (IOException e) {
throw new RuntimeException("Failure to loading template files into i/o stream");
}

this._userSpecifiedAttributesList =
Expand All @@ -77,6 +76,19 @@ public List<String> getRequiredConfigList() {
return this._userSpecifiedAttributesList;
}

/**
* For backward compatibility
* @param userProps
* @return
*/
public Properties getResolvedConfigAsProperties(Properties userProps) {
Properties jobPropsWithPotentialTemplate = userProps;
if (jobPropsWithPotentialTemplate.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
jobPropsWithPotentialTemplate = TemplateUtils.mergeTemplateWithUserCustomizedFile(this.configTemplate, userProps);
}
return jobPropsWithPotentialTemplate;
}

/**
* Return the combine configuration of template and user customized attributes.
* @return
Expand All @@ -85,7 +97,8 @@ public List<String> getRequiredConfigList() {
public Config getResolvedConfig(Properties userProps) {
Config jobPropsWithPotentialTemplate = ConfigFactory.parseProperties(userProps);
if (userProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
jobPropsWithPotentialTemplate = TemplateUtils.mergeTemplateWithUserCustomizedFile(this.configTemplate, userProps);
jobPropsWithPotentialTemplate = ConfigFactory.parseProperties(
TemplateUtils.mergeTemplateWithUserCustomizedFile(this.configTemplate, userProps));
}
return jobPropsWithPotentialTemplate;
}
Expand Down
10 changes: 4 additions & 6 deletions gobblin-utility/src/main/java/gobblin/util/TemplateUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
*/
package gobblin.util;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import java.util.Properties;

import gobblin.configuration.ConfigurationKeys;
Expand All @@ -22,17 +20,17 @@ public class TemplateUtils {
/**
* create a complete property file based on the given template
*/
public static Config mergeTemplateWithUserCustomizedFile(Properties template, Properties userCustomized) {
Properties cleanedTemplate = new Properties(template);
public static Properties mergeTemplateWithUserCustomizedFile(Properties template, Properties userCustomized) {
Properties cleanedTemplate = template;
if (cleanedTemplate.containsKey(ConfigurationKeys.REQUIRED_ATRRIBUTES_LIST)) {
cleanedTemplate.remove(ConfigurationKeys.REQUIRED_ATRRIBUTES_LIST);
}

Properties cleanedUserCustomized = new Properties(userCustomized) ;
Properties cleanedUserCustomized = userCustomized ;
if (cleanedUserCustomized.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
cleanedUserCustomized.remove(ConfigurationKeys.JOB_TEMPLATE_PATH);
}

return ConfigFactory.parseProperties(PropertiesUtils.combineProperties(cleanedTemplate, cleanedUserCustomized));
return PropertiesUtils.combineProperties(cleanedTemplate, cleanedUserCustomized);
}
}
101 changes: 101 additions & 0 deletions gobblin-utility/src/test/java/gobblin/util/TemplateTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use
* this file except in compliance with the License. You may obtain a copy of the
* License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied.
*/
package gobblin.util;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.List;
import java.util.Properties;

import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import gobblin.configuration.ConfigurationKeys;
/**
* Testing the functions for reading template merging template with user-specified attributes.
* 1. Reading the template configuration, testing size or something
* 2. Testing the required attributes result.
*/
@Test(groups = {"gobblin.runtime"})
public class TemplateTest {

// For template inside resource folder.
private File userCustomizedFile;
private File jobConfigDir;
private Properties userProp ;

@BeforeClass
public void setUp()
throws IOException, URISyntaxException {

// Creating userCustomized stuff
this.jobConfigDir =
Files.createTempDirectory(String.format("gobblin-test_%s_job-conf", this.getClass().getSimpleName())).toFile();
FileUtils.forceDeleteOnExit(this.jobConfigDir);

//User specified file content:
this.userProp = new Properties();
userProp.setProperty("a", "1");
userProp.setProperty("b", "2");
userProp.setProperty("job.template", "templates/test.template");

// User specified file's name : /[jobConfigDirName]/user.attr
userProp.store(new FileWriter(new File(this.jobConfigDir, "user.attr")), "");
}

@Test
public void testRequiredAttrList() {
Properties jobProps = this.userProp;

List<String> requiredConfigList = (new SimpleGeneralJobTemplate(
jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH))).getRequiredConfigList();
Assert.assertEquals(requiredConfigList.size(), 3);
Assert.assertEquals(requiredConfigList.get(0), "required0");
Assert.assertEquals(requiredConfigList.get(1), "required1");
Assert.assertEquals(requiredConfigList.get(2), "required2");
}

// Testing the resolving of userCustomized attributes and template is correct.
@Test
public void testResolvingConfig()
throws IOException {
Properties jobProps = this.userProp ;
Assert.assertEquals(jobProps.size(), 3);
jobProps = (new SimpleGeneralJobTemplate(
jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH))).getResolvedConfigAsProperties(jobProps);
// Remove job.template in userSpecified file and required.attribute in template
Assert.assertEquals(jobProps.size(), 5);

Properties targetResolvedJobProps = new Properties() ;
targetResolvedJobProps.setProperty("a", "1");
targetResolvedJobProps.setProperty("b", "2");
targetResolvedJobProps.setProperty("attr1","x");
targetResolvedJobProps.setProperty("attr2","y");
targetResolvedJobProps.setProperty("attr3","z");

Assert.assertEquals(targetResolvedJobProps, jobProps);
}

@AfterClass
public void tearDown()
throws IOException {
if (this.jobConfigDir != null) {
FileUtils.forceDelete(this.jobConfigDir);
}
}
}
4 changes: 4 additions & 0 deletions gobblin-utility/src/test/resources/templates/test.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
required.attribute=required0,required1,required2
attr1=x
attr2=y
attr3=z
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# Dragged directly from wikipedia.pull

job.name=PullFromWikipedia
job.group=Wikipedia
job.description=A getting started example for Gobblin
Expand All @@ -21,5 +19,5 @@ writer.partitioner.class=gobblin.example.wikipedia.WikipediaPartitioner

data.publisher.type=gobblin.publisher.BaseDataPublisher

required.attribute=required1,required2,required3
required.attribute=required0,required1,required2

0 comments on commit baa4219

Please sign in to comment.